diff options
Diffstat (limited to 'extras')
108 files changed, 9265 insertions, 3467 deletions
diff --git a/extras/Makefile.am b/extras/Makefile.am index 68637724a48..983f014cca6 100644 --- a/extras/Makefile.am +++ b/extras/Makefile.am @@ -1,33 +1,58 @@ addonexecdir = $(GLUSTERFS_LIBEXECDIR) -addonexec_SCRIPTS = peer_add_secret_pub +addonexec_SCRIPTS = +if WITH_SERVER +addonexec_SCRIPTS += peer_add_secret_pub if USE_SYSTEMD addonexec_SCRIPTS += mount-shared-storage.sh endif +endif EditorModedir = $(docdir) EditorMode_DATA = glusterfs-mode.el glusterfs.vim SUBDIRS = init.d systemd benchmarking hook-scripts $(OCF_SUBDIR) LinuxRPM \ - $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils + $(GEOREP_EXTRAS_SUBDIR) snap_scheduler firewalld cliutils python \ + ganesha confdir = $(sysconfdir)/glusterfs +if WITH_SERVER conf_DATA = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ - logger.conf.example glusterfs-georep-logrotate group-virt.example group-metadata-cache group-gluster-block group-nl-cache + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-distributed-virt group-samba +endif voldir = $(sysconfdir)/glusterfs -vol_DATA = glusterd.vol +vol_DATA = thin-arbiter/thin-arbiter.vol +if WITH_SERVER +vol_DATA += glusterd.vol +endif + scriptsdir = $(datadir)/glusterfs/scripts -scripts_SCRIPTS = post-upgrade-script-for-quota.sh \ +scripts_SCRIPTS = thin-arbiter/setup-thin-arbiter.sh +if WITH_SERVER +scripts_SCRIPTS += post-upgrade-script-for-quota.sh \ pre-upgrade-script-for-quota.sh stop-all-gluster-processes.sh +if USE_SYSTEMD +scripts_SCRIPTS += control-cpu-load.sh +scripts_SCRIPTS += control-mem.sh +endif +endif -EXTRA_DIST = $(conf_DATA) specgen.scm glusterfs-mode.el glusterfs.vim \ - migrate-unify-to-distribute.sh backend-xattr-sanitize.sh backend-cleanup.sh \ - disk_usage_sync.sh clear_xattrs.sh glusterd-sysconfig glusterd.vol \ - post-upgrade-script-for-quota.sh pre-upgrade-script-for-quota.sh \ - command-completion/gluster.bash command-completion/Makefile \ - command-completion/README stop-all-gluster-processes.sh clang-checker.sh \ - mount-shared-storage.sh +EXTRA_DIST = glusterfs-logrotate gluster-rsyslog-7.2.conf gluster-rsyslog-5.8.conf \ + logger.conf.example glusterfs-georep-logrotate group-virt.example \ + group-metadata-cache group-gluster-block group-nl-cache \ + group-db-workload group-samba specgen.scm glusterfs-mode.el glusterfs.vim \ + migrate-unify-to-distribute.sh backend-xattr-sanitize.sh \ + backend-cleanup.sh disk_usage_sync.sh clear_xattrs.sh \ + glusterd-sysconfig glusterd.vol post-upgrade-script-for-quota.sh \ + pre-upgrade-script-for-quota.sh command-completion/gluster.bash \ + command-completion/Makefile command-completion/README \ + stop-all-gluster-processes.sh clang-checker.sh mount-shared-storage.sh \ + control-cpu-load.sh control-mem.sh group-distributed-virt \ + thin-arbiter/thin-arbiter.vol thin-arbiter/setup-thin-arbiter.sh +if WITH_SERVER install-data-local: if [ -n "$(tmpfilesdir)" ]; then \ $(mkdir_p) $(DESTDIR)$(tmpfilesdir); \ @@ -43,3 +68,10 @@ install-data-local: $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/gluster-block $(INSTALL_DATA) $(top_srcdir)/extras/group-nl-cache \ $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/nl-cache + $(INSTALL_DATA) $(top_srcdir)/extras/group-db-workload \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/db-workload + $(INSTALL_DATA) $(top_srcdir)/extras/group-distributed-virt \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/distributed-virt + $(INSTALL_DATA) $(top_srcdir)/extras/group-samba \ + $(DESTDIR)$(GLUSTERD_WORKDIR)/groups/samba +endif diff --git a/extras/benchmarking/glfs-bm.c b/extras/benchmarking/glfs-bm.c index dc717f33c16..f7f5873f84d 100644 --- a/extras/benchmarking/glfs-bm.c +++ b/extras/benchmarking/glfs-bm.c @@ -25,365 +25,338 @@ #include <sys/time.h> struct state { - char need_op_write:1; - char need_op_read:1; + char need_op_write : 1; + char need_op_read : 1; - char need_iface_fileio:1; - char need_iface_xattr:1; + char need_iface_fileio : 1; + char need_iface_xattr : 1; - char need_mode_posix:1; + char need_mode_posix : 1; - char prefix[512]; - long int count; + char prefix[512]; + long int count; - size_t block_size; + size_t block_size; - char *specfile; + char *specfile; - long int io_size; + long int io_size; }; - -#define MEASURE(func, arg) measure (func, #func, arg) - +#define MEASURE(func, arg) measure(func, #func, arg) void -tv_difference (struct timeval *tv_stop, - struct timeval *tv_start, - struct timeval *tv_diff) +tv_difference(struct timeval *tv_stop, struct timeval *tv_start, + struct timeval *tv_diff) { - if (tv_stop->tv_usec < tv_start->tv_usec) { - tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; - tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); - } else { - tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; - tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; - } + if (tv_stop->tv_usec < tv_start->tv_usec) { + tv_diff->tv_usec = (tv_stop->tv_usec + 1000000) - tv_start->tv_usec; + tv_diff->tv_sec = (tv_stop->tv_sec - 1 - tv_start->tv_sec); + } else { + tv_diff->tv_usec = tv_stop->tv_usec - tv_start->tv_usec; + tv_diff->tv_sec = tv_stop->tv_sec - tv_start->tv_sec; + } } - void -measure (int (*func)(struct state *state), - char *func_name, struct state *state) +measure(int (*func)(struct state *state), char *func_name, struct state *state) { - struct timeval tv_start, tv_stop, tv_diff; - state->io_size = 0; - long int count; + struct timeval tv_start, tv_stop, tv_diff; + state->io_size = 0; + long int count; - gettimeofday (&tv_start, NULL); - count = func (state); - gettimeofday (&tv_stop, NULL); + gettimeofday(&tv_start, NULL); + count = func(state); + gettimeofday(&tv_stop, NULL); - tv_difference (&tv_stop, &tv_start, &tv_diff); + tv_difference(&tv_stop, &tv_start, &tv_diff); - fprintf (stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", - func_name, count, state->io_size, - tv_diff.tv_sec, tv_diff.tv_usec); + fprintf(stdout, "%s: count=%ld, size=%ld, time=%ld:%ld\n", func_name, count, + state->io_size, tv_diff.tv_sec, tv_diff.tv_usec); } - static error_t -parse_opts (int key, char *arg, - struct argp_state *_state) +parse_opts(int key, char *arg, struct argp_state *_state) { - struct state *state = _state->input; + struct state *state = _state->input; - switch (key) - { + switch (key) { case 'o': - if (strcasecmp (arg, "read") == 0) { - state->need_op_write = 0; - state->need_op_read = 1; - } else if (strcasecmp (arg, "write") == 0) { - state->need_op_write = 1; - state->need_op_read = 0; - } else if (strcasecmp (arg, "both") == 0) { - state->need_op_write = 1; - state->need_op_read = 1; - } else { - fprintf (stderr, "unknown op: %s\n", arg); - return -1; - } - break; + if (strcasecmp(arg, "read") == 0) { + state->need_op_write = 0; + state->need_op_read = 1; + } else if (strcasecmp(arg, "write") == 0) { + state->need_op_write = 1; + state->need_op_read = 0; + } else if (strcasecmp(arg, "both") == 0) { + state->need_op_write = 1; + state->need_op_read = 1; + } else { + fprintf(stderr, "unknown op: %s\n", arg); + return -1; + } + break; case 'i': - if (strcasecmp (arg, "fileio") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 0; - } else if (strcasecmp (arg, "xattr") == 0) { - state->need_iface_fileio = 0; - state->need_iface_xattr = 1; - } else if (strcasecmp (arg, "both") == 0) { - state->need_iface_fileio = 1; - state->need_iface_xattr = 1; - } else { - fprintf (stderr, "unknown interface: %s\n", arg); - return -1; - } - break; - case 'b': - { - size_t block_size = atoi (arg); - if (!block_size) { - fprintf (stderr, "incorrect size: %s\n", arg); - return -1; - } - state->block_size = block_size; - } - break; + if (strcasecmp(arg, "fileio") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 0; + } else if (strcasecmp(arg, "xattr") == 0) { + state->need_iface_fileio = 0; + state->need_iface_xattr = 1; + } else if (strcasecmp(arg, "both") == 0) { + state->need_iface_fileio = 1; + state->need_iface_xattr = 1; + } else { + fprintf(stderr, "unknown interface: %s\n", arg); + return -1; + } + break; + case 'b': { + size_t block_size = atoi(arg); + if (!block_size) { + fprintf(stderr, "incorrect size: %s\n", arg); + return -1; + } + state->block_size = block_size; + } break; case 's': - state->specfile = strdup (arg); - break; + state->specfile = strdup(arg); + break; case 'p': - fprintf (stderr, "using prefix: %s\n", arg); - strncpy (state->prefix, arg, 512); - break; - case 'c': - { - long count = atol (arg); - if (!count) { - fprintf (stderr, "incorrect count: %s\n", arg); - return -1; - } - state->count = count; - } - break; + fprintf(stderr, "using prefix: %s\n", arg); + strncpy(state->prefix, arg, 512); + break; + case 'c': { + long count = atol(arg); + if (!count) { + fprintf(stderr, "incorrect count: %s\n", arg); + return -1; + } + state->count = count; + } break; case ARGP_KEY_NO_ARGS: - break; + break; case ARGP_KEY_ARG: - break; - } + break; + } - return 0; + return 0; } int -do_mode_posix_iface_fileio_write (struct state *state) +do_mode_posix_iface_fileio_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_CREAT|O_WRONLY, 00600); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = write (fd, block, state->block_size); - if (ret != state->block_size) { - fprintf (stderr, "write (%s) => %d/%s\n", filename, ret, - strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + + sprintf(filename, "%s.%06ld", state->prefix, i); + + fd = open(filename, O_CREAT | O_WRONLY, 00600); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; + } + ret = write(fd, block, state->block_size); + if (ret != state->block_size) { + fprintf(stderr, "write (%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_fileio_read (struct state *state) +do_mode_posix_iface_fileio_read(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - - for (i=0; i<state->count; i++) { - int fd = -1; - char filename[512]; - - sprintf (filename, "%s.%06ld", state->prefix, i); - - fd = open (filename, O_RDONLY); - if (fd == -1) { - fprintf (stderr, "open(%s) => %s\n", filename, strerror (errno)); - break; - } - ret = read (fd, block, state->block_size); - if (ret == -1) { - fprintf (stderr, "read(%s) => %d/%s\n", filename, ret, strerror (errno)); - close (fd); - break; - } - close (fd); - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + + for (i = 0; i < state->count; i++) { + int fd = -1; + char filename[512]; + + sprintf(filename, "%s.%06ld", state->prefix, i); + + fd = open(filename, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "open(%s) => %s\n", filename, strerror(errno)); + break; } + ret = read(fd, block, state->block_size); + if (ret == -1) { + fprintf(stderr, "read(%s) => %d/%s\n", filename, ret, + strerror(errno)); + close(fd); + break; + } + close(fd); + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_fileio (struct state *state) +do_mode_posix_iface_fileio(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_fileio_write, state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_fileio_write, state); - if (state->need_op_read) - MEASURE (do_mode_posix_iface_fileio_read, state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_fileio_read, state); - return 0; + return 0; } - int -do_mode_posix_iface_xattr_write (struct state *state) +do_mode_posix_iface_xattr_write(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lsetxattr (dname, key, block, state->block_size, 0); - - if (ret != 0) { - fprintf (stderr, "lsetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += state->block_size; + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; + + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + + for (i = 0; i < state->count; i++) { + char key[512]; + + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); + + ret = lsetxattr(dname, key, block, state->block_size, 0); + + if (ret != 0) { + fprintf(stderr, "lsetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; } + state->io_size += state->block_size; + } - free (dirc); - free (basec); + free(dirc); + free(basec); - return i; + return i; } - int -do_mode_posix_iface_xattr_read (struct state *state) +do_mode_posix_iface_xattr_read(struct state *state) { - long int i; - int ret = -1; - char block[state->block_size]; - char *dname = NULL, *dirc = NULL; - char *bname = NULL, *basec = NULL; - - dirc = strdup (state->prefix); - basec = strdup (state->prefix); - dname = dirname (dirc); - bname = basename (basec); - - for (i=0; i<state->count; i++) { - char key[512]; - - sprintf (key, "glusterfs.file.%s.%06ld", bname, i); - - ret = lgetxattr (dname, key, block, state->block_size); - - if (ret < 0) { - fprintf (stderr, "lgetxattr (%s, %s, %p) => %s\n", - dname, key, block, strerror (errno)); - break; - } - state->io_size += ret; + long int i; + int ret = -1; + char block[state->block_size]; + char *dname = NULL, *dirc = NULL; + char *bname = NULL, *basec = NULL; + + dirc = strdup(state->prefix); + basec = strdup(state->prefix); + dname = dirname(dirc); + bname = basename(basec); + + for (i = 0; i < state->count; i++) { + char key[512]; + + sprintf(key, "glusterfs.file.%s.%06ld", bname, i); + + ret = lgetxattr(dname, key, block, state->block_size); + + if (ret < 0) { + fprintf(stderr, "lgetxattr (%s, %s, %p) => %s\n", dname, key, block, + strerror(errno)); + break; } + state->io_size += ret; + } - return i; + return i; } - int -do_mode_posix_iface_xattr (struct state *state) +do_mode_posix_iface_xattr(struct state *state) { - if (state->need_op_write) - MEASURE (do_mode_posix_iface_xattr_write, state); + if (state->need_op_write) + MEASURE(do_mode_posix_iface_xattr_write, state); - if (state->need_op_read) - MEASURE (do_mode_posix_iface_xattr_read, state); + if (state->need_op_read) + MEASURE(do_mode_posix_iface_xattr_read, state); - return 0; + return 0; } int -do_mode_posix (struct state *state) +do_mode_posix(struct state *state) { - if (state->need_iface_fileio) - do_mode_posix_iface_fileio (state); + if (state->need_iface_fileio) + do_mode_posix_iface_fileio(state); - if (state->need_iface_xattr) - do_mode_posix_iface_xattr (state); + if (state->need_iface_xattr) + do_mode_posix_iface_xattr(state); - return 0; + return 0; } - int -do_actions (struct state *state) +do_actions(struct state *state) { - if (state->need_mode_posix) - do_mode_posix (state); + if (state->need_mode_posix) + do_mode_posix(state); - return 0; + return 0; } static struct argp_option options[] = { - {"op", 'o', "OPERATIONS", 0, - "WRITE|READ|BOTH - defaults to BOTH"}, - {"iface", 'i', "INTERFACE", 0, - "FILEIO|XATTR|BOTH - defaults to FILEIO"}, - {"block", 'b', "BLOCKSIZE", 0, - "<NUM> - defaults to 4096"}, - {"specfile", 's', "SPECFILE", 0, - "absolute path to specfile"}, - {"prefix", 'p', "PREFIX", 0, - "filename prefix"}, - {"count", 'c', "COUNT", 0, - "number of files"}, - {0, 0, 0, 0, 0} -}; + {"op", 'o', "OPERATIONS", 0, "WRITE|READ|BOTH - defaults to BOTH"}, + {"iface", 'i', "INTERFACE", 0, "FILEIO|XATTR|BOTH - defaults to FILEIO"}, + {"block", 'b', "BLOCKSIZE", 0, "<NUM> - defaults to 4096"}, + {"specfile", 's', "SPECFILE", 0, "absolute path to specfile"}, + {"prefix", 'p', "PREFIX", 0, "filename prefix"}, + {"count", 'c', "COUNT", 0, "number of files"}, + {0, 0, 0, 0, 0}}; -static struct argp argp = { - options, - parse_opts, - "tool", - "tool to benchmark small file performance" -}; +static struct argp argp = {options, parse_opts, "tool", + "tool to benchmark small file performance"}; int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - struct state state = {0, }; + struct state state = { + 0, + }; - state.need_op_write = 1; - state.need_op_read = 1; + state.need_op_write = 1; + state.need_op_read = 1; - state.need_iface_fileio = 1; - state.need_iface_xattr = 0; + state.need_iface_fileio = 1; + state.need_iface_xattr = 0; - state.need_mode_posix = 1; + state.need_mode_posix = 1; - state.block_size = 4096; + state.block_size = 4096; - strcpy (state.prefix, "tmpfile"); - state.count = 1048576; + strcpy(state.prefix, "tmpfile"); + state.count = 1048576; - if (argp_parse (&argp, argc, argv, 0, 0, &state) != 0) { - fprintf (stderr, "argp_parse() failed\n"); - return 1; - } + if (argp_parse(&argp, argc, argv, 0, 0, &state) != 0) { + fprintf(stderr, "argp_parse() failed\n"); + return 1; + } - do_actions (&state); + do_actions(&state); - return 0; + return 0; } diff --git a/extras/benchmarking/rdd.c b/extras/benchmarking/rdd.c index a667c6a1d65..efc9d342a37 100644 --- a/extras/benchmarking/rdd.c +++ b/extras/benchmarking/rdd.c @@ -20,633 +20,586 @@ #define TWO_POWER(power) (2UL << (power)) -#define RDD_INTEGER_VALUE ((TWO_POWER ((sizeof (int) * 8))) - 1) +#define RDD_INTEGER_VALUE ((TWO_POWER((sizeof(int) * 8))) - 1) #ifndef UNIX_PATH_MAX #define UNIX_PATH_MAX 108 #endif #define UNIT_KB 1024ULL -#define UNIT_MB UNIT_KB*1024ULL -#define UNIT_GB UNIT_MB*1024ULL -#define UNIT_TB UNIT_GB*1024ULL -#define UNIT_PB UNIT_TB*1024ULL +#define UNIT_MB UNIT_KB * 1024ULL +#define UNIT_GB UNIT_MB * 1024ULL +#define UNIT_TB UNIT_GB * 1024ULL +#define UNIT_PB UNIT_TB * 1024ULL -#define UNIT_KB_STRING "KB" -#define UNIT_MB_STRING "MB" -#define UNIT_GB_STRING "GB" -#define UNIT_TB_STRING "TB" -#define UNIT_PB_STRING "PB" +#define UNIT_KB_STRING "KB" +#define UNIT_MB_STRING "MB" +#define UNIT_GB_STRING "GB" +#define UNIT_TB_STRING "TB" +#define UNIT_PB_STRING "PB" struct rdd_file { - char path[UNIX_PATH_MAX]; - struct stat st; - int fd; + char path[UNIX_PATH_MAX]; + struct stat st; + int fd; }; struct rdd_config { - long iters; - long max_ops_per_seq; - size_t max_bs; - size_t min_bs; - int thread_count; - pthread_t *threads; - pthread_barrier_t barrier; - pthread_mutex_t lock; - struct rdd_file in_file; - struct rdd_file out_file; - ssize_t file_size; + long iters; + long max_ops_per_seq; + size_t max_bs; + size_t min_bs; + int thread_count; + pthread_t *threads; + pthread_barrier_t barrier; + pthread_mutex_t lock; + struct rdd_file in_file; + struct rdd_file out_file; + ssize_t file_size; }; static struct rdd_config rdd_config; enum rdd_keys { - RDD_MIN_BS_KEY = 1, - RDD_MAX_BS_KEY, + RDD_MIN_BS_KEY = 1, + RDD_MAX_BS_KEY, }; static error_t -rdd_parse_opts (int key, char *arg, - struct argp_state *_state) +rdd_parse_opts(int key, char *arg, struct argp_state *_state) { - switch (key) { - case 'o': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "output file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.out_file.path, arg, len); - } - break; - - case 'i': - { - int len = 0; - len = strlen (arg); - if (len > UNIX_PATH_MAX) { - fprintf (stderr, "input file name too long (%s)\n", - arg); - return -1; - } - - strncpy (rdd_config.in_file.path, arg, len); - rdd_config.in_file.path[len] = '\0'; - } - break; - - case 'f': - { - char *tmp = NULL; - unsigned long long fs = 0; - if (string2bytesize (arg, &fs) == -1) { - fprintf (stderr, "invalid argument for file size " - "(%s)\n", arg); - return -1; - } - - rdd_config.file_size = fs; - } - break; - - case RDD_MIN_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for minimum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.min_bs = bs; - } - break; - - case RDD_MAX_BS_KEY: - { - char *tmp = NULL; - long bs = 0; - bs = strtol (arg, &tmp, 10); - if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { - fprintf (stderr, "invalid argument for maximum block" - "size (%s)\n", arg); - return -1; - } - - rdd_config.max_bs = bs; - } - break; - - case 'r': - { - char *tmp = NULL; - long iters = 0; - iters = strtol (arg, &tmp, 10); - if ((iters == LONG_MAX) || - (iters == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for iterations" - "(%s)\n", arg); - return -1; - } - - rdd_config.iters = iters; - } - break; - - case 'm': - { - char *tmp = NULL; - long max_ops = 0; - max_ops = strtol (arg, &tmp, 10); - if ((max_ops == LONG_MAX) || - (max_ops == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for max-ops" - "(%s)\n", arg); - return -1; - } + switch (key) { + case 'o': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "output file name too long (%s)\n", arg); + return -1; + } - rdd_config.max_ops_per_seq = max_ops; - } - break; + strncpy(rdd_config.out_file.path, arg, len); + } break; - case 't': - { - char *tmp = NULL; - long threads = 0; - threads = strtol (arg, &tmp, 10); - if ((threads == LONG_MAX) || - (threads == LONG_MIN) || - (tmp && *tmp)) { - fprintf (stderr, "invalid argument for thread count" - "(%s)\n", arg); - return -1; - } + case 'i': { + int len = 0; + len = strlen(arg); + if (len > UNIX_PATH_MAX) { + fprintf(stderr, "input file name too long (%s)\n", arg); + return -1; + } + + strncpy(rdd_config.in_file.path, arg, len); + rdd_config.in_file.path[len] = '\0'; + } break; + + case 'f': { + char *tmp = NULL; + unsigned long long fs = 0; + if (string2bytesize(arg, &fs) == -1) { + fprintf(stderr, + "invalid argument for file size " + "(%s)\n", + arg); + return -1; + } + + rdd_config.file_size = fs; + } break; + + case RDD_MIN_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for minimum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.min_bs = bs; + } break; + + case RDD_MAX_BS_KEY: { + char *tmp = NULL; + long bs = 0; + bs = strtol(arg, &tmp, 10); + if ((bs == LONG_MAX) || (bs == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for maximum block" + "size (%s)\n", + arg); + return -1; + } + + rdd_config.max_bs = bs; + } break; + + case 'r': { + char *tmp = NULL; + long iters = 0; + iters = strtol(arg, &tmp, 10); + if ((iters == LONG_MAX) || (iters == LONG_MIN) || (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for iterations" + "(%s)\n", + arg); + return -1; + } + + rdd_config.iters = iters; + } break; + + case 'm': { + char *tmp = NULL; + long max_ops = 0; + max_ops = strtol(arg, &tmp, 10); + if ((max_ops == LONG_MAX) || (max_ops == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for max-ops" + "(%s)\n", + arg); + return -1; + } + + rdd_config.max_ops_per_seq = max_ops; + } break; + + case 't': { + char *tmp = NULL; + long threads = 0; + threads = strtol(arg, &tmp, 10); + if ((threads == LONG_MAX) || (threads == LONG_MIN) || + (tmp && *tmp)) { + fprintf(stderr, + "invalid argument for thread count" + "(%s)\n", + arg); + return -1; + } - rdd_config.thread_count = threads; - } - break; + rdd_config.thread_count = threads; + } break; case ARGP_KEY_NO_ARGS: - break; + break; case ARGP_KEY_ARG: - break; + break; case ARGP_KEY_END: - if (_state->argc == 1) { - argp_usage (_state); - } + if (_state->argc == 1) { + argp_usage(_state); + } + } - } - - return 0; + return 0; } int -string2bytesize (const char *str, unsigned long long *n) +string2bytesize(const char *str, unsigned long long *n) { - unsigned long long value = 0ULL; - char *tail = NULL; - int old_errno = 0; - const char *s = NULL; - - if (str == NULL || n == NULL) - { - errno = EINVAL; - return -1; - } - - for (s = str; *s != '\0'; s++) - { - if (isspace (*s)) - { - continue; - } - if (*s == '-') - { - return -1; - } - break; + unsigned long long value = 0ULL; + char *tail = NULL; + int old_errno = 0; + const char *s = NULL; + + if (str == NULL || n == NULL) { + errno = EINVAL; + return -1; + } + + for (s = str; *s != '\0'; s++) { + if (isspace(*s)) { + continue; } - - old_errno = errno; - errno = 0; - value = strtoull (str, &tail, 10); - - if (errno == ERANGE || errno == EINVAL) - { - return -1; + if (*s == '-') { + return -1; } - - if (errno == 0) - { - errno = old_errno; + break; + } + + old_errno = errno; + errno = 0; + value = strtoull(str, &tail, 10); + + if (errno == ERANGE || errno == EINVAL) { + return -1; + } + + if (errno == 0) { + errno = old_errno; + } + + if (tail[0] != '\0') { + if (strcasecmp(tail, UNIT_KB_STRING) == 0) { + value *= UNIT_KB; + } else if (strcasecmp(tail, UNIT_MB_STRING) == 0) { + value *= UNIT_MB; + } else if (strcasecmp(tail, UNIT_GB_STRING) == 0) { + value *= UNIT_GB; + } else if (strcasecmp(tail, UNIT_TB_STRING) == 0) { + value *= UNIT_TB; + } else if (strcasecmp(tail, UNIT_PB_STRING) == 0) { + value *= UNIT_PB; } - if (tail[0] != '\0') - { - if (strcasecmp (tail, UNIT_KB_STRING) == 0) - { - value *= UNIT_KB; - } - else if (strcasecmp (tail, UNIT_MB_STRING) == 0) - { - value *= UNIT_MB; - } - else if (strcasecmp (tail, UNIT_GB_STRING) == 0) - { - value *= UNIT_GB; - } - else if (strcasecmp (tail, UNIT_TB_STRING) == 0) - { - value *= UNIT_TB; - } - else if (strcasecmp (tail, UNIT_PB_STRING) == 0) - { - value *= UNIT_PB; - } - - else - { - return -1; - } + else { + return -1; } + } - *n = value; + *n = value; - return 0; + return 0; } static struct argp_option rdd_options[] = { - {"if", 'i', "INPUT_FILE", 0, "input-file"}, - {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, - {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, - {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, - "Minimum block size in bytes (defaults to 1024)"}, - {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, - "Maximum block size in bytes (defaults to 4096)"}, - {"iters", 'r', "ITERS", 0, - "Number of read-write sequences (defaults to 1000000)"}, - {"max-ops", 'm', "MAXOPS", 0, - "maximum number of read-writes to be performed in a sequence (defaults to 1)"}, - {"file-size", 'f', "FILESIZE", 0, - "the size of the file which will be created and upon it I/O will be done" - " (defaults to 100MB"}, - {0, 0, 0, 0, 0} -}; + {"if", 'i', "INPUT_FILE", 0, "input-file"}, + {"of", 'o', "OUTPUT_FILE", 0, "output-file"}, + {"threads", 't', "COUNT", 0, "number of threads to spawn (defaults to 2)"}, + {"min-bs", RDD_MIN_BS_KEY, "MIN_BLOCK_SIZE", 0, + "Minimum block size in bytes (defaults to 1024)"}, + {"max-bs", RDD_MAX_BS_KEY, "MAX_BLOCK_SIZE", 0, + "Maximum block size in bytes (defaults to 4096)"}, + {"iters", 'r', "ITERS", 0, + "Number of read-write sequences (defaults to 1000000)"}, + {"max-ops", 'm', "MAXOPS", 0, + "maximum number of read-writes to be performed in a sequence (defaults to " + "1)"}, + {"file-size", 'f', "FILESIZE", 0, + "the size of the file which will be created and upon it I/O will be done" + " (defaults to 100MB"}, + {0, 0, 0, 0, 0}}; static struct argp argp = { - rdd_options, - rdd_parse_opts, - "", - "random dd - tool to do a sequence of random block-sized continuous" - "read writes starting at a random offset" -}; - + rdd_options, rdd_parse_opts, "", + "random dd - tool to do a sequence of random block-sized continuous" + "read writes starting at a random offset"}; static void -rdd_default_config (void) +rdd_default_config(void) { - char *tmp_path = "rdd.in"; - - rdd_config.thread_count = 2; - rdd_config.iters = 1000000; - rdd_config.max_bs = 4096; - rdd_config.min_bs = 1024; - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; - rdd_config.max_ops_per_seq = 1; - strncpy (rdd_config.in_file.path, tmp_path, strlen (tmp_path)); - rdd_config.file_size = 104857600; - - return; + char *tmp_path = "rdd.in"; + + rdd_config.thread_count = 2; + rdd_config.iters = 1000000; + rdd_config.max_bs = 4096; + rdd_config.min_bs = 1024; + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; + rdd_config.max_ops_per_seq = 1; + strncpy(rdd_config.in_file.path, tmp_path, strlen(tmp_path)); + rdd_config.file_size = 104857600; + + return; } - static char -rdd_valid_config (void) +rdd_valid_config(void) { - char ret = 1; - int fd = -1; + char ret = 1; + int fd = -1; - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd == -1 && (errno != ENOENT)) { - fprintf (stderr, "open: (%s)", strerror (errno)); - ret = 0; - goto out; - } - close (fd); - - if (rdd_config.min_bs > rdd_config.max_bs) { - fprintf (stderr, "minimum blocksize %ld is greater than the " - "maximum blocksize %ld", rdd_config.min_bs, - rdd_config.max_bs); - ret = 0; - goto out; - } + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd == -1 && (errno != ENOENT)) { + fprintf(stderr, "open: (%s)", strerror(errno)); + ret = 0; + goto out; + } + close(fd); + + if (rdd_config.min_bs > rdd_config.max_bs) { + fprintf(stderr, + "minimum blocksize %ld is greater than the " + "maximum blocksize %ld", + rdd_config.min_bs, rdd_config.max_bs); + ret = 0; + goto out; + } - if (strlen (rdd_config.out_file.path) == 0) { - sprintf (rdd_config.out_file.path, "%s.rddout", - rdd_config.in_file.path); - } + if (strlen(rdd_config.out_file.path) == 0) { + sprintf(rdd_config.out_file.path, "%s.rddout", rdd_config.in_file.path); + } out: - return ret; + return ret; } - static void * -rdd_read_write (void *arg) +rdd_read_write(void *arg) { - int i = 0, ret = 0; - size_t bs = 0; - off_t offset = 0; - long rand = 0; - long max_ops = 0; - char *buf = NULL; - - buf = calloc (1, rdd_config.max_bs); - if (!buf) { - fprintf (stderr, "calloc failed (%s)\n", strerror (errno)); + int i = 0, ret = 0; + size_t bs = 0; + off_t offset = 0; + long rand = 0; + long max_ops = 0; + char *buf = NULL; + + buf = calloc(1, rdd_config.max_bs); + if (!buf) { + fprintf(stderr, "calloc failed (%s)\n", strerror(errno)); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.iters; i++) { + pthread_mutex_lock(&rdd_config.lock); + { + int bytes = 0; + rand = random(); + + if (rdd_config.min_bs == rdd_config.max_bs) { + bs = rdd_config.max_bs; + } else { + bs = rdd_config.min_bs + + (rand % (rdd_config.max_bs - rdd_config.min_bs)); + } + + offset = rand % rdd_config.in_file.st.st_size; + max_ops = rand % rdd_config.max_ops_per_seq; + if (!max_ops) { + max_ops++; + } + + ret = lseek(rdd_config.in_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); ret = -1; - goto out; - } + goto unlock; + } - for (i = 0; i < rdd_config.iters; i++) - { - pthread_mutex_lock (&rdd_config.lock); - { - int bytes = 0; - rand = random (); - - if (rdd_config.min_bs == rdd_config.max_bs) { - bs = rdd_config.max_bs; - } else { - bs = rdd_config.min_bs + - (rand % - (rdd_config.max_bs - - rdd_config.min_bs)); - } - - offset = rand % rdd_config.in_file.st.st_size; - max_ops = rand % rdd_config.max_ops_per_seq; - if (!max_ops) { - max_ops ++; - } - - ret = lseek (rdd_config.in_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - ret = lseek (rdd_config.out_file.fd, offset, SEEK_SET); - if (ret != offset) { - fprintf (stderr, "lseek failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - while (max_ops--) - { - bytes = read (rdd_config.in_file.fd, buf, bs); - if (!bytes) { - break; - } - - if (bytes == -1) { - fprintf (stderr, "read failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - - if (write (rdd_config.out_file.fd, buf, bytes) - != bytes) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - ret = -1; - goto unlock; - } - } + ret = lseek(rdd_config.out_file.fd, offset, SEEK_SET); + if (ret != offset) { + fprintf(stderr, "lseek failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + while (max_ops--) { + bytes = read(rdd_config.in_file.fd, buf, bs); + if (!bytes) { + break; } - unlock: - pthread_mutex_unlock (&rdd_config.lock); - if (ret == -1) { - goto out; + + if (bytes == -1) { + fprintf(stderr, "read failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; + } + + if (write(rdd_config.out_file.fd, buf, bytes) != bytes) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + ret = -1; + goto unlock; } - ret = 0; + } } + unlock: + pthread_mutex_unlock(&rdd_config.lock); + if (ret == -1) { + goto out; + } + ret = 0; + } out: - free (buf); - pthread_barrier_wait (&rdd_config.barrier); + free(buf); + pthread_barrier_wait(&rdd_config.barrier); - return NULL; + return NULL; } static void -cleanup (void) +cleanup(void) { - close (rdd_config.in_file.fd); - close (rdd_config.out_file.fd); - rdd_config.in_file.fd = rdd_config.out_file.fd = -1; + close(rdd_config.in_file.fd); + close(rdd_config.out_file.fd); + rdd_config.in_file.fd = rdd_config.out_file.fd = -1; } static int -check_and_create (void) +check_and_create(void) { - int ret = -1; - char buf[4096] = {0,}; - struct stat stbuf = {0,}; - int fd[2] = {-1,}; - size_t total_size = -1; - - total_size = rdd_config.file_size; - - ret = stat (rdd_config.in_file.path, &stbuf); - if (ret == -1 && (errno != ENOENT)) + int ret = -1; + char buf[4096] = { + 0, + }; + struct stat stbuf = { + 0, + }; + int fd[2] = { + -1, + }; + size_t total_size = -1; + + total_size = rdd_config.file_size; + + ret = stat(rdd_config.in_file.path, &stbuf); + if (ret == -1 && (errno != ENOENT)) + goto out; + + fd[1] = open(rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC); + if (fd[1] == -1) + goto out; + + fd[0] = open("/dev/urandom", O_RDONLY); + if (fd[0] == -1) + goto out; + + while (total_size > 0) { + if (total_size >= 4096) { + ret = read(fd[0], buf, 4096); + if (ret == -1) goto out; - - fd[1] = open (rdd_config.in_file.path, O_CREAT | O_WRONLY | O_TRUNC); - if (fd[1] == -1) + ret = write(fd[1], buf, 4096); + if (ret == -1) goto out; - - fd[0] = open ("/dev/urandom", O_RDONLY); - if (fd[0] == -1) + total_size = total_size - 4096; + } else { + ret = read(fd[0], buf, total_size); + if (ret == -1) goto out; - - while (total_size > 0) { - if (total_size >= 4096) { - ret = read (fd[0], buf, 4096); - if (ret == -1) - goto out; - ret = write (fd[1], buf, 4096); - if (ret == -1) - goto out; - total_size = total_size - 4096; - } else { - ret = read (fd[0], buf, total_size); - if (ret == -1) - goto out; - ret = write (fd[1], buf, total_size); - if (ret == -1) - goto out; - total_size = total_size - total_size; - } - + ret = write(fd[1], buf, total_size); + if (ret == -1) + goto out; + total_size = total_size - total_size; } + } - ret = 0; + ret = 0; out: - if (fd[0] > 0) - close (fd[0]); - if (fd[1] > 0) - close (fd[1]); - return ret; + if (fd[0] > 0) + close(fd[0]); + if (fd[1] > 0) + close(fd[1]); + return ret; } static int -rdd_spawn_threads (void) +rdd_spawn_threads(void) { - int i = 0, ret = -1, fd = -1; - char buf[4096]; - - ret = check_and_create (); - if (ret == -1) - goto out; - - fd = open (rdd_config.in_file.path, O_RDONLY); - if (fd < 0) { - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - ret = fstat (fd, &rdd_config.in_file.st); - if (ret != 0) { - close (fd); - fprintf (stderr, "cannot stat %s (%s)\n", - rdd_config.in_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.in_file.fd = fd; - - fd = open (rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC, - S_IRWXU | S_IROTH); - if (fd < 0) { - close (rdd_config.in_file.fd); - rdd_config.in_file.fd = -1; - fprintf (stderr, "cannot open %s (%s)\n", - rdd_config.out_file.path, strerror (errno)); - ret = -1; - goto out; - } - rdd_config.out_file.fd = fd; - - while ((ret = read (rdd_config.in_file.fd, buf, 4096)) > 0) { - if (write (rdd_config.out_file.fd, buf, ret) != ret) { - fprintf (stderr, "write failed (%s)\n", - strerror (errno)); - cleanup (); - ret = -1; - goto out; - } - } - - rdd_config.threads = calloc (rdd_config.thread_count, - sizeof (pthread_t)); - if (rdd_config.threads == NULL) { - fprintf (stderr, "calloc() failed (%s)\n", strerror (errno)); - - ret = -1; - cleanup (); - goto out; - } - - ret = pthread_barrier_init (&rdd_config.barrier, NULL, - rdd_config.thread_count + 1); - if (ret != 0) { - fprintf (stderr, "pthread_barrier_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - cleanup (); - ret = -1; - goto out; + int i = 0, ret = -1, fd = -1; + char buf[4096]; + + ret = check_and_create(); + if (ret == -1) + goto out; + + fd = open(rdd_config.in_file.path, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + ret = fstat(fd, &rdd_config.in_file.st); + if (ret != 0) { + close(fd); + fprintf(stderr, "cannot stat %s (%s)\n", rdd_config.in_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.in_file.fd = fd; + + fd = open(rdd_config.out_file.path, O_WRONLY | O_CREAT | O_TRUNC, + S_IRWXU | S_IROTH); + if (fd < 0) { + close(rdd_config.in_file.fd); + rdd_config.in_file.fd = -1; + fprintf(stderr, "cannot open %s (%s)\n", rdd_config.out_file.path, + strerror(errno)); + ret = -1; + goto out; + } + rdd_config.out_file.fd = fd; + + while ((ret = read(rdd_config.in_file.fd, buf, 4096)) > 0) { + if (write(rdd_config.out_file.fd, buf, ret) != ret) { + fprintf(stderr, "write failed (%s)\n", strerror(errno)); + cleanup(); + ret = -1; + goto out; } - - ret = pthread_mutex_init (&rdd_config.lock, NULL); + } + + rdd_config.threads = calloc(rdd_config.thread_count, sizeof(pthread_t)); + if (rdd_config.threads == NULL) { + fprintf(stderr, "calloc() failed (%s)\n", strerror(errno)); + + ret = -1; + cleanup(); + goto out; + } + + ret = pthread_barrier_init(&rdd_config.barrier, NULL, + rdd_config.thread_count + 1); + if (ret != 0) { + fprintf(stderr, "pthread_barrier_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + cleanup(); + ret = -1; + goto out; + } + + ret = pthread_mutex_init(&rdd_config.lock, NULL); + if (ret != 0) { + fprintf(stderr, "pthread_mutex_init() failed (%s)\n", strerror(ret)); + + free(rdd_config.threads); + pthread_barrier_destroy(&rdd_config.barrier); + cleanup(); + ret = -1; + goto out; + } + + for (i = 0; i < rdd_config.thread_count; i++) { + ret = pthread_create(&rdd_config.threads[i], NULL, rdd_read_write, + NULL); if (ret != 0) { - fprintf (stderr, "pthread_mutex_init() failed (%s)\n", - strerror (ret)); - - free (rdd_config.threads); - pthread_barrier_destroy (&rdd_config.barrier); - cleanup (); - ret = -1; - goto out; - } - - for (i = 0; i < rdd_config.thread_count; i++) - { - ret = pthread_create (&rdd_config.threads[i], NULL, - rdd_read_write, NULL); - if (ret != 0) { - fprintf (stderr, "pthread_create failed (%s)\n", - strerror (errno)); - exit (1); - } + fprintf(stderr, "pthread_create failed (%s)\n", strerror(errno)); + exit(1); } + } out: - return ret; + return ret; } static void -rdd_wait_for_completion (void) +rdd_wait_for_completion(void) { - pthread_barrier_wait (&rdd_config.barrier); + pthread_barrier_wait(&rdd_config.barrier); } - int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; + int ret = -1; - rdd_default_config (); + rdd_default_config(); - ret = argp_parse (&argp, argc, argv, 0, 0, NULL); - if (ret != 0) { - ret = -1; - fprintf (stderr, "%s: argp_parse() failed\n", argv[0]); - goto err; - } + ret = argp_parse(&argp, argc, argv, 0, 0, NULL); + if (ret != 0) { + ret = -1; + fprintf(stderr, "%s: argp_parse() failed\n", argv[0]); + goto err; + } - if (!rdd_valid_config ()) { - ret = -1; - fprintf (stderr, "%s: configuration validation failed\n", - argv[0]); - goto err; - } + if (!rdd_valid_config()) { + ret = -1; + fprintf(stderr, "%s: configuration validation failed\n", argv[0]); + goto err; + } - ret = rdd_spawn_threads (); - if (ret != 0) { - fprintf (stderr, "%s: spawning threads failed\n", argv[0]); - goto err; - } + ret = rdd_spawn_threads(); + if (ret != 0) { + fprintf(stderr, "%s: spawning threads failed\n", argv[0]); + goto err; + } - rdd_wait_for_completion (); + rdd_wait_for_completion(); err: - return ret; + return ret; } diff --git a/extras/cliutils/README.md b/extras/cliutils/README.md index ccb60802c3d..309beb1ca25 100644 --- a/extras/cliutils/README.md +++ b/extras/cliutils/README.md @@ -81,7 +81,7 @@ to address the following issues Create a file in `$LIBEXEC/glusterfs/peer_message.py` with following content. - #!/usr/bin/env python + #!/usr/bin/python3 from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok class NodeHello(Cmd): @@ -149,7 +149,7 @@ Now users can use `gluster-message` instead of calling Following example uses prettytable library, which can be installed using `pip install prettytable` or `dnf install python-prettytable` - #!/usr/bin/env python + #!/usr/bin/python3 from prettytable import PrettyTable from gluster.cliutils import Cmd, runcli, execute_in_peers, node_output_ok @@ -221,7 +221,7 @@ required.(Under `%files` section) - gluster-mountbroker http://review.gluster.org/14544 - gluster-eventsapi http://review.gluster.org/14248 - gluster-georep-sshkey http://review.gluster.org/14732 -- gluster-restapi https://github.com/aravindavk/glusterfs-restapi +- gluster-restapi https://github.com/gluster/restapi ## Limitations/TODOs - Not yet possible to create CLI without any subcommand, For example diff --git a/extras/cliutils/__init__.py b/extras/cliutils/__init__.py index 9c930982be0..8765cc85099 100644 --- a/extras/cliutils/__init__.py +++ b/extras/cliutils/__init__.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- # Reexporting the utility funcs and classes -from cliutils import (runcli, - sync_file_to_peers, - execute_in_peers, - execute, - node_output_ok, - node_output_notok, - output_error, - oknotok, - yesno, - get_node_uuid, - Cmd, - GlusterCmdException, - set_common_args_func) +from .cliutils import (runcli, + sync_file_to_peers, + execute_in_peers, + execute, + node_output_ok, + node_output_notok, + output_error, + oknotok, + yesno, + get_node_uuid, + Cmd, + GlusterCmdException, + set_common_args_func) # This will be useful when `from cliutils import *` diff --git a/extras/cliutils/cliutils.py b/extras/cliutils/cliutils.py index d805ac6d100..55fbaf56704 100644 --- a/extras/cliutils/cliutils.py +++ b/extras/cliutils/cliutils.py @@ -20,7 +20,14 @@ _common_args_func = lambda p: True class GlusterCmdException(Exception): - pass + def __init__(self, message): + self.message = message + try: + # Python 3 + super().__init__(message) + except TypeError: + # Python 2 + super(GlusterCmdException, self).__init__(message) def get_node_uuid(): @@ -71,7 +78,8 @@ def node_output_notok(message): def execute(cmd): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) out, err = p.communicate() return p.returncode, out, err diff --git a/extras/collect-system-stats.sh b/extras/collect-system-stats.sh new file mode 100755 index 00000000000..865e70bbc11 --- /dev/null +++ b/extras/collect-system-stats.sh @@ -0,0 +1,52 @@ +#!/bin/bash +################################################################################ +# Usage: collect-system-stats.sh <delay-in-seconds> +# This script starts sar/top/iostat/vmstat processes which collect system stats +# with the interval <delay-in-seconds> given as argument to the script. When +# the script is stopped either by entering any input or Ctrl+C the list of +# files where output is captured will be printed on the screen which can be +# observed to find any problems/bottlenecks. +############################################################################### + +function stop_processes { + echo "Stopping the monitoring processes" + echo "sar pid:$sar_pid", "top pid: $top_pid", "iostat pid: $iostat_pid", "vmstat pid: $vmstat_pid" + kill "$sar_pid" "$top_pid" "$iostat_pid" "$vmstat_pid" + echo "Files created: ${timestamp}-network.out, ${timestamp}-top.out, ${timestamp}-iostat.out, ${timestamp}-vmstat.out" +} + +function check_dependent_commands_exist() +{ + declare -a arr=("sar" "top" "iostat" "vmstat") + for i in "${arr[@]}" + do + if ! command -v "$i" > /dev/null 2>&1 + then + echo "ERROR: '$i' command is not found" + exit 1 + fi + done + +} + +case "$1" in + ''|*[!0-9]*) echo "Usage: $0 <delay-between-successive-metrics-collection-in-seconds>"; exit 1 ;; + *) interval="$1" ;; +esac + +timestamp=$(date +"%s") + +check_dependent_commands_exist +sar -n DEV "$interval" > "${timestamp}"-network.out & +sar_pid="$!" +top -bHd "$interval" > "${timestamp}"-top.out & +top_pid="$!" +iostat -Ntkdx "$interval" > "${timestamp}"-iostat.out & +iostat_pid="$!" +vmstat -t "$interval" > "${timestamp}"-vmstat.out & +vmstat_pid="$!" +echo "Started sar, vmstat, iostat, top for collecting stats" + + +trap stop_processes EXIT +read -r -p "Press anything and ENTER to exit"; diff --git a/extras/command-completion/gluster.bash b/extras/command-completion/gluster.bash index 680ecd964d5..73d16098875 100644 --- a/extras/command-completion/gluster.bash +++ b/extras/command-completion/gluster.bash @@ -26,28 +26,28 @@ GLUSTER_TOP_SUBOPTIONS2=" " GLUSTER_TOP_OPTIONS=" {open - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {read - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {write - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {opendir - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {readdir - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {clear - [ $TOP_SUBOPTIONS1 ] + [ $GLUSTER_TOP_SUBOPTIONS1 ] }, {read-perf - [ $TOP_SUBOPTIONS2 ] + [ $GLUSTER_TOP_SUBOPTIONS2 ] }, {write-perf - [ $TOP_SUBOPTIONS2 ] + [ $GLUSTER_TOP_SUBOPTIONS2 ] } " @@ -282,16 +282,16 @@ _gluster_throw () { exit } -declare FINAL_LIST='' -declare LIST='' -declare -i TOP=0 +declare GLUSTER_FINAL_LIST='' +declare GLUSTER_LIST='' +declare -i GLUSTER_TOP=0 _gluster_push () { - TOP=$((TOP + 1)) - return $TOP + GLUSTER_TOP=$((GLUSTER_TOP + 1)) + return $GLUSTER_TOP } _gluster_pop () { - TOP=$((TOP - 1)) - return $TOP + GLUSTER_TOP=$((GLUSTER_TOP - 1)) + return $GLUSTER_TOP } _gluster_goto_end () @@ -333,7 +333,7 @@ _gluster_form_list () top=$? read -r key if [ "X$cur_word" == "X" -o "${cur_word:0:1}" == "${key:0:1}" -o "${key:0:1}" == "_" ]; then - LIST="$LIST $key" + GLUSTER_LIST="$GLUSTER_LIST $key" fi _gluster_goto_end $top @@ -452,10 +452,10 @@ _gluster_parse () elif [ "$token" == '{' ]; then read -r tmp_token - LIST="$tmp_token" + GLUSTER_LIST="$tmp_token" fi - echo $LIST + echo $GLUSTER_LIST } _gluster_handle_list () @@ -479,12 +479,12 @@ _gluster_handle_list () _gluster_completion () { - FINAL_LIST=`echo $GLUSTER_COMMAND_TREE | \ + GLUSTER_FINAL_LIST=`echo $GLUSTER_COMMAND_TREE | \ egrep -ao --color=never "([A-Za-z0-9_.-]+)|[[:space:]]+|." | \ egrep -v --color=never "^[[:space:]]*$" | \ _gluster_parse` - ARG="FINAL_LIST" + ARG="GLUSTER_FINAL_LIST" _gluster_handle_list $ARG ${COMP_WORDS[COMP_CWORD]} return } diff --git a/extras/control-cpu-load.sh b/extras/control-cpu-load.sh new file mode 100755 index 00000000000..52dcf62fd9f --- /dev/null +++ b/extras/control-cpu-load.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +USAGE="This script provides a utility to control CPU utilization for any +gluster daemon.In this, we use cgroup framework to configure CPU quota +for a process(like selfheal daemon). Before running this script, make +sure that daemon is running.Every time daemon restarts, it is required +to rerun this command to set CPU quota on new daemon process id. +User can enter any value between 10 to 100 for CPU quota. +Recommended value of quota period is 25. 25 means, kernel will allocate +25 ms period to this group of tasks in every 100 ms period. This 25ms +could be considered as the maximum percentage of CPU quota daemon can take. +This value will be reflected on CPU usage of "top" command.If provided pid +is the only process and no other process is in competition to get CPU, more + than 25% could be allocated to daemon to speed up the process." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h or --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/cpu,cpuacct/system.slice/glusterd.service" +echo "Enter gluster daemon pid for which you want to control CPU." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ]; then + echo "pid ${daemon_pid} is attached with glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/cpu.cfs_quota_us` + qval=$((val / 1000)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with quota value ${qval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new quota value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach ${daemon_pid} with new ${cgroup_name} cgroup Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter quota value in range [10,100]: " + +read quota_value +if expr ${quota_value} + 0 > /dev/null 2>&1 ;then + if [ ${quota_value} -lt 10 ] || [ ${quota_value} -gt 100 ]; then + echo "Entered quota value is not correct,it should be in the range ." + echo "10-100. Ideal value is 25." + echo "Rerun the sript with correct value." + exit 1 + else + echo "Entered quota value is $quota_value" + fi +else + echo "Entered quota value is not numeric so Rerun the script." + exit 1 +fi + +quota_value=$((quota_value * 1000)) +echo "Setting $quota_value to cpu.cfs_quota_us for gluster_cgroup." +echo ${quota_value} > ${LOC}/${cgroup_name}/cpu.cfs_quota_us + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -w ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/control-mem.sh b/extras/control-mem.sh new file mode 100755 index 00000000000..91b36f8107a --- /dev/null +++ b/extras/control-mem.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +USAGE="This commands provides a utility to control MEMORY utilization for any +gluster daemon.In this, we use cgroup framework to configure MEMORY limit for +a process. Before running this script, make sure that daemon is running.Every +time daemon restarts, it is required to rerun this command to set memory limit +(in bytes) on new daemon process id.User can enter any value between 100 +(in Mega bytes) to 8000000000000 for Memory limit in Mega bytes. +Memory limit value is depends on how much maximum memory user wants to restrict +for specific daemon process.If a process will try to consume memore more than +configured value then cgroup will hang/sleep this task and to resume the task +rerun the script with new increase memory limit value ." + +if [ $# -ge 1 ]; then + case $1 in + -h|--help) echo " " "$USAGE" | sed -r -e 's/^[ ]+//g' + exit 0; + ;; + *) echo "Please Provide correct input for script." + echo "For help correct options are -h of --help." + exit 1; + ;; + esac +fi + +DIR_EXIST=0 +LOC="/sys/fs/cgroup/memory/system.slice/glusterd.service" +echo "Enter Any gluster daemon pid for that you want to control MEMORY." +read daemon_pid + +if expr ${daemon_pid} + 0 > /dev/null 2>&1 ;then + CHECK_PID=$(pgrep -f gluster | grep ${daemon_pid}) + if [ -z "${CHECK_PID}" ]; then + echo "No daemon is running or pid ${daemon_pid} does not match." + echo "with running gluster processes." + exit 1 + fi +else + echo "Entered daemon_pid is not numeric so Rerun the script." + exit 1 +fi + + +if [ -f ${LOC}/tasks ]; then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/tasks) + if [ ${CHECK_CGROUP} ] ;then + echo "pid ${daemon_pid} is attached with default glusterd.service cgroup." + fi +fi + +cgroup_name=cgroup_gluster_${daemon_pid} +if [ -f ${LOC}/${cgroup_name}/tasks ];then + CHECK_CGROUP=$(grep ${daemon_pid} ${LOC}/${cgroup_name}/tasks) + if [ ${CHECK_CGROUP} ]; then + val=`cat ${LOC}/${cgroup_name}/memory.limit_in_bytes` + mval=$((val / 1024 / 1024)) + echo "pid ${daemon_pid} is already attached ${cgroup_name} with mem value ${mval}." + echo "Press n if you don't want to reassign ${daemon_pid} with new mem value." + DIR_EXIST=1 + else + echo "pid ${daemon_pid} is not attached with ${cgroup_name}." + fi +fi + +read -p "If you want to continue the script to attach daeomon with new cgroup. Press (y/n)?" choice +case "$choice" in + y|Y ) echo "yes";; + n|N ) echo "no";exit;; + * ) echo "invalid";exit;; +esac + +systemctl set-property glusterd.service CPUShares=1024 + +if [ ${DIR_EXIST} -eq 0 ];then + echo "Creating child cgroup directory '${cgroup_name} cgroup' for glusterd.service." + mkdir -p ${LOC}/${cgroup_name} + if [ ! -f ${LOC}/${cgroup_name}/tasks ];then + echo "Not able to create ${LOC}/${cgroup_name} directory so exit." + exit 1 + fi +fi + +echo "Enter Memory value in Mega bytes [100,8000000000000]: " + +read mem_value +if expr ${mem_value} + 0 > /dev/null 2>&1 ;then + if [ ${mem_value} -lt 100 ] || [ ${mem_value} -gt 8000000000000 ]; then + echo "Entered memory value is not correct,it should be in the range ." + echo "100-8000000000000, Rerun the script with correct value ." + exit 1 + else + echo "Entered memory limit value is ${mem_value}." + fi +else + echo "Entered memory value is not numeric so Rerun the script." + exit 1 +fi + +mem_value=$(($mem_value * 1024 * 1024)) +if [ ${DIR_EXIST} -eq 0 ];then + echo "Setting ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + #Set memory value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +else + #Increase mem_value to memory.memsw.limit_in_bytes + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.memsw.limit_in_bytes + echo "Increase ${mem_value} to memory.limit_in_bytes for ${LOC}/${cgroup_name}." + echo ${mem_value} > ${LOC}/${cgroup_name}/memory.limit_in_bytes + # disable oom_control so that kernel will not send kill signal to the + # task once limit has reached + echo 1 > ${LOC}/${cgroup_name}/memory.oom_control +fi + +if ps -T -p ${daemon_pid} | grep gluster > /dev/null; then + for thid in `ps -T -p ${daemon_pid} | grep -v SPID | awk -F " " '{print $2}'`; + do + echo ${thid} > ${LOC}/${cgroup_name}/tasks ; + done + if cat /proc/${daemon_pid}/cgroup | grep -iw ${cgroup_name} > /dev/null; then + echo "Tasks are attached successfully specific to ${daemon_pid} to ${cgroup_name}." + else + echo "Tasks are not attached successfully." + fi +fi diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py index 2dbf470eafb..983868c04db 100755 --- a/extras/create_new_xlator/generate_xlator.py +++ b/extras/create_new_xlator/generate_xlator.py @@ -1,4 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 + +from __future__ import print_function import os import re import sys @@ -34,11 +36,11 @@ def get_error_arg(type_str): def get_param(names, types): # Convert two separate tuples to one of (name, type) sub-tuples. - as_tuples = zip(types, names) + as_tuples = list(zip(types, names)) # Convert each sub-tuple into a "type name" string. - as_strings = map(string.join, as_tuples) + as_strings = [' '.join(item) for item in as_tuples] # Join all of those into one big string. - return string.join(as_strings, ",\n\t") + return ',\n\t'.join(as_strings) def generate(tmpl, name, table): @@ -54,18 +56,18 @@ def generate(tmpl, name, table): sdict = {} #Parameters are (t1, var1), (t2, var2)... #Args are (var1, var2,...) - sdict["@WIND_ARGS@"] = string.join(w_arg_names, ", ") - sdict["@UNWIND_ARGS@"] = string.join(u_arg_names, ", ") - sdict["@ERROR_ARGS@"] = string.join(map(get_error_arg, u_arg_types), ", ") + sdict["@WIND_ARGS@"] = ', '.join(w_arg_names) + sdict["@UNWIND_ARGS@"] = ', '.join(u_arg_names) + sdict["@ERROR_ARGS@"] = ', '.join(list(map(get_error_arg, u_arg_types))) sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types) sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types) sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types) sdict["@NAME@"] = name sdict["@FOP_PREFIX@"] = fop_prefix - sdict["@RET_TYPE@"] = string.join(ret_type, "") - sdict["@RET_VAR@"] = string.join(ret_var, "") + sdict["@RET_TYPE@"] = ''.join(ret_type) + sdict["@RET_VAR@"] = ''.join(ret_var) - for old, new in sdict.iteritems(): + for old, new in sdict.items(): tmpl = tmpl.replace(old, new) # TBD: reindent/reformat the result for maximum readability. return tmpl @@ -74,42 +76,44 @@ def generate(tmpl, name, table): def gen_xlator(): xl = open(src_dir_path+"/"+xl_name+".c", 'w+') - print >> xl, COPYRIGHT - print >> xl, fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@", - xl_name) + print(COPYRIGHT, file=xl) + print(fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@", + xl_name), file=xl) #Generate cbks and fops for fop in ops: - print >> xl, generate(fragments["CBK_TEMPLATE"], fop, ops) - print >> xl, generate(fragments["FOP_TEMPLATE"], fop, ops) + print(generate(fragments["CBK_TEMPLATE"], fop, ops), file=xl) + print(generate(fragments["FOP_TEMPLATE"], fop, ops), file=xl) for cbk in xlator_cbks: - print >> xl, generate(fragments["FUNC_TEMPLATE"], cbk, - xlator_cbks) + print(generate(fragments["FUNC_TEMPLATE"], cbk, + xlator_cbks), file=xl) for dops in xlator_dumpops: - print >> xl, generate(fragments["FUNC_TEMPLATE"], dops, - xlator_dumpops) - - print >> xl, fragments["XLATOR_METHODS"] + print(generate(fragments["FUNC_TEMPLATE"], dops, + xlator_dumpops), file=xl) #Generate fop table - print >> xl, "struct xlator_fops fops = {" + print("struct xlator_fops fops = {", file=xl) for fop in ops: - print >> xl, " .{0:20} = {1}_{2},".format(fop, fop_prefix, fop) - print >> xl, "};" + print(" .{0:20} = {1}_{2},".format(fop, fop_prefix, fop), file=xl) + print("};", file=xl) #Generate xlator_cbks table - print >> xl, "struct xlator_cbks cbks = {" + print("struct xlator_cbks cbks = {", file=xl) for cbk in xlator_cbks: - print >> xl, " .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk) - print >> xl, "};" + print(" .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk), file=xl) + print("};", file=xl) #Generate xlator_dumpops table - print >> xl, "struct xlator_dumpops dumpops = {" + print("struct xlator_dumpops dumpops = {", file=xl) for dops in xlator_dumpops: - print >> xl, " .{0:20} = {1}_{2},".format(dops, fop_prefix, dops) - print >> xl, "};" + print(" .{0:20} = {1}_{2},".format(dops, fop_prefix, dops), file=xl) + print("};", file=xl) + + xlator_methods = fragments["XLATOR_METHODS"].replace("@XL_NAME@", xl_name) + xlator_methods = xlator_methods.replace("@FOP_PREFIX@", fop_prefix) + print(xlator_methods, file=xl) xl.close() @@ -122,38 +126,37 @@ def create_dir_struct(): def gen_header_files(): upname = xl_name_no_hyphen.upper() h = open(src_dir_path+"/"+xl_name+".h", 'w+') - print >> h, COPYRIGHT + print(COPYRIGHT, file=h) txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname) - txt2 = fragments["INCLUDE_IN_HEADER_FILE"].replace("@XL_NAME@", xl_name) - txt = txt.replace("@INCLUDE_SECT@",txt2) - print >> h, txt + txt = txt.replace("@XL_NAME@", xl_name) + print(txt, file=h) h.close() h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+') - print >> h, COPYRIGHT - txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES") - txt = txt.replace("@INCLUDE_SECT@", '#include "mem-types.h"') - print >> h, txt + print(COPYRIGHT, file=h) + txt = fragments["MEM_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix) + print(txt, file=h) h.close() h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+') - print >> h, COPYRIGHT - txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES") - txt = txt.replace("@INCLUDE_SECT@", '') - print >> h, txt + print(COPYRIGHT, file=h) + txt = fragments["MSG_HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES") + txt = txt.replace("@FOP_PREFIX@", fop_prefix.upper()) + print(txt, file=h) h.close() def gen_makefiles(): m = open(dir_path+"/Makefile.am", 'w+') - print >> m, "SUBDIRS = src\n\nCLEANFILES =" + print("SUBDIRS = src\n\nCLEANFILES =", file=m) m.close() m = open(src_dir_path+"/Makefile.am", 'w+') txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name) txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen) - txt = txt.replace("@XL_TYPE@",xlator_type) - print >> m, txt + txt = txt.replace("@XL_TYPE@", xlator_type) + print(txt, file=m) m.close() def get_copyright (): @@ -166,8 +169,8 @@ def load_fragments (): cur_value = "" result = {} basepath = os.path.abspath(os.path.dirname(__file__)) - fragpath = basepath + "/new-xlator-tmpl.c" - for line in open(fragpath,"r").readlines(): + fragpath = basepath + "/new-xlator.c.tmpl" + for line in open(fragpath, "r").readlines(): m = pragma_re.search(line) if m: if cur_symbol: @@ -183,7 +186,7 @@ def load_fragments (): if __name__ == '__main__': if len(sys.argv) < 3: - print "USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>" + print("USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>") sys.exit(0) xl_name = sys.argv[2] diff --git a/extras/create_new_xlator/new-xlator-tmpl.c b/extras/create_new_xlator/new-xlator-tmpl.c deleted file mode 100644 index ac08f3732a7..00000000000 --- a/extras/create_new_xlator/new-xlator-tmpl.c +++ /dev/null @@ -1,89 +0,0 @@ -#pragma fragment CBK_TEMPLATE -int32_t -@FOP_PREFIX@_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno, @UNWIND_PARAMS@) -{ - STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno, - @UNWIND_ARGS@); - return 0; -} - -#pragma fragment COMMENT -If you are generating the leaf xlators, remove the STACK_WIND -and replace the @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary - -#pragma fragment FOP_TEMPLATE -int32_t -@FOP_PREFIX@_@NAME@ (call_frame_t *frame, xlator_t *this, - @WIND_PARAMS@) -{ - STACK_WIND (frame, @FOP_PREFIX@_@NAME@_cbk, - FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@, - @WIND_ARGS@); - return 0; -err: - STACK_UNWIND_STRICT (@NAME@, frame, -1, errno, - @ERROR_ARGS@); - return 0; -} - -#pragma fragment FUNC_TEMPLATE -@RET_TYPE@ -@FOP_PREFIX@_@NAME@ (@FUNC_PARAMS@) -{ - return @RET_VAR@; -} - -#pragma fragment CP -/* - * Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com> - * This file is part of GlusterFS. - * - * This file is licensed to you under your choice of the GNU Lesser - * General Public License, version 3 or any later version (LGPLv3 or - * later), or the GNU General Public License, version 2 (GPLv2), in all - * cases as published by the Free Software Foundation. - */ - -#pragma fragment INCLUDE_IN_SRC_FILE -#include "@XL_NAME@.h" - -#pragma fragment INCLUDE_IN_HEADER_FILE -#include "@XL_NAME@-mem-types.h" -#include "@XL_NAME@-messages.h" -#include "glusterfs.h" -#include "xlator.h" -#include "defaults.h" - -#pragma fragment XLATOR_METHODS -int32_t -init (xlator_t *this) -{ - return 0; -} - -void -fini (xlator_t *this) -{ - return; -} - -int32_t -reconfigure (xlator_t *this, dict_t *dict) -{ - return 0; -} - -int -notify (xlator_t *this, int event, void *data, ...) -{ - return default_notify (this, event, data); -} - -#pragma fragment HEADER_FMT -#ifndef __@HFL_NAME@_H__ -#define __@HFL_NAME@_H__ - -@INCLUDE_SECT@ - -#endif /* __@HFL_NAME@_H__ */ diff --git a/extras/create_new_xlator/new-xlator.c.tmpl b/extras/create_new_xlator/new-xlator.c.tmpl new file mode 100644 index 00000000000..fe9735bfcf1 --- /dev/null +++ b/extras/create_new_xlator/new-xlator.c.tmpl @@ -0,0 +1,151 @@ +#pragma fragment CBK_TEMPLATE +int32_t @FOP_PREFIX@_@NAME@_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, @UNWIND_PARAMS@) +{ + STACK_UNWIND_STRICT(@NAME@, frame, op_ret, op_errno, @UNWIND_ARGS@); + return 0; +} + +#pragma fragment COMMENT +If you are generating the leaf xlators, remove the STACK_WIND and replace the + @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary + +#pragma fragment FOP_TEMPLATE + int32_t @FOP_PREFIX@_@NAME@(call_frame_t *frame, xlator_t *this, @WIND_PARAMS@) +{ + STACK_WIND(frame, @FOP_PREFIX@_@NAME@_cbk, FIRST_CHILD(this), + FIRST_CHILD(this)->fops->@NAME@, @WIND_ARGS@); + return 0; +err: + STACK_UNWIND_STRICT(@NAME@, frame, -1, errno, @ERROR_ARGS@); + return 0; +} + +#pragma fragment FUNC_TEMPLATE +@RET_TYPE@ @FOP_PREFIX@_@NAME@(@FUNC_PARAMS@) +{ + return @RET_VAR@; +} + +#pragma fragment CP +/* + * Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com> + * This file is part of GlusterFS. + * + * This file is licensed to you under your choice of the GNU Lesser + * General Public License, version 3 or any later version (LGPLv3 or + * later), or the GNU General Public License, version 2 (GPLv2), in all + * cases as published by the Free Software Foundation. + */ + +#pragma fragment INCLUDE_IN_SRC_FILE +#include "@XL_NAME@.h" + +#pragma fragment XLATOR_METHODS + +static int32_t @FOP_PREFIX@_init(xlator_t *this) +{ + return 0; +} + +static void @FOP_PREFIX@_fini(xlator_t *this) +{ + return; +} + +static int32_t @FOP_PREFIX@_reconfigure(xlator_t *this, dict_t *dict) +{ + return 0; +} + +static int @FOP_PREFIX@_notify(xlator_t *this, int event, void *data, ...) +{ + return default_notify(this, event, data); +} + +static int32_t @FOP_PREFIX@_mem_acct_init(xlator_t *this) +{ + int ret = -1; + + ret = xlator_mem_acct_init(this, gf_@FOP_PREFIX@_mt_end + 1); + return ret; +} + +static int32_t @FOP_PREFIX@_dump_metrics(xlator_t *this, int fd) +{ + return 0; +} + +struct volume_options @FOP_PREFIX@_options[] = { + /*{ .key = {""}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "", + .op_version = {GD_OP_VERSION_}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT, + .tags = {""}, + .description = "", + .category = GF_EXPERIMENTAL, + }, + { .key = {NULL} }, + */ +}; + +xlator_api_t xlator_api = { + .init = @FOP_PREFIX@_init, + .fini = @FOP_PREFIX@_fini, + .notify = @FOP_PREFIX@_notify, + .reconfigure = @FOP_PREFIX@_reconfigure, + .mem_acct_init = @FOP_PREFIX@_mem_acct_init, + .dump_metrics = @FOP_PREFIX@_dump_metrics, + .op_version = {GD_OP_VERSION_}, + .dumpops = &@FOP_PREFIX@_dumpops, + .fops = &@FOP_PREFIX@_fops, + .cbks = &@FOP_PREFIX @_cbks, + .options = @FOP_PREFIX@_options, + .identifier = "@XL_NAME@", + .category = GF_EXPERIMENTAL, +}; +#pragma fragment HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include "@XL_NAME@-mem-types.h" +#include "@XL_NAME@-messages.h" +#include <glusterfs/glusterfs.h> +#include <glusterfs/xlator.h> +#include <glusterfs/defaults.h> + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MEM_HEADER_FMT +#ifndef __ @HFL_NAME@_H__ +#define __ @HFL_NAME@_H__ + +#include <glusterfs/mem-types.h> + +enum gf_mdc_mem_types_ { + gf_@FOP_PREFIX@_mt_ = gf_common_mt_end + 1, + gf_@FOP_PREFIX@_mt_end +}; + +#endif /* __@HFL_NAME@_H__ */ + +#pragma fragment MSG_HEADER_FMT +#ifndef __@HFL_NAME@_H__ +#define __@HFL_NAME@_H__ + +#include <glusterfs/glfs-message-id.h> + +/* To add new message IDs, append new identifiers at the end of the list. + * + * Never remove a message ID. If it's not used anymore, you can rename it or + * leave it as it is, but not delete it. This is to prevent reutilization of + * IDs by other messages. + * + * The component name must match one of the entries defined in + * glfs-message-id.h. + */ + +GLFS_MSGID(@FOP_PREFIX@, @FOP_PREFIX@_MSG_NO_MEMORY); + +#endif /* __@HFL_NAME@_H__ */ diff --git a/extras/devel-tools/print-backtrace.sh b/extras/devel-tools/print-backtrace.sh index 873b6bc692d..33fbae288bc 100755 --- a/extras/devel-tools/print-backtrace.sh +++ b/extras/devel-tools/print-backtrace.sh @@ -20,6 +20,8 @@ # Usage with source install: # print-packtrace.sh none bt-file.txt +function version_compare() { test $(echo $1|awk -F '.' '{print $1 $2 $3}') -gt $(echo $2|awk -F '.' '{print $1 $2 $3}'); } + function Usage() { echo -e "Usage:\n\t$0 { none | <debuginfo-rpm> } <backtrace-file>" @@ -38,9 +40,11 @@ if [ ! $debuginfo_rpm ] || [ ! $backtrace_file ]; then exit 1 fi -if [ $debuginfo_rpm != "none" ] && [ ! -f $debuginfo_rpm ]; then - echo "no such rpm file: $debuginfo_rpm" - exit 1 +if [ $debuginfo_rpm != "none" ]; then + if [ ! -f $debuginfo_rpm ]; then + echo "no such rpm file: $debuginfo_rpm" + exit 1 + fi fi if [ ! -f $backtrace_file ]; then @@ -48,11 +52,14 @@ if [ ! -f $backtrace_file ]; then exit 1 fi -if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then - echo "file does not look like an rpm: $debuginfo_rpm" - exit 1 +if [ "$debuginfo_rpm" != "none" ]; then + if ! file $debuginfo_rpm | grep RPM >/dev/null 2>&1 ; then + echo "file does not look like an rpm: $debuginfo_rpm" + exit 1 + fi fi +cpio_version=$(cpio --version|grep cpio|cut -f 2 -d ')'|sed -e 's/^[[:space:]]*//') rpm_name="" debuginfo_path="" debuginfo_extension="" @@ -66,8 +73,21 @@ if [ $debuginfo_rpm != "none" ]; then exit 1 fi mkdir -p $rpm_name - rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name - + if version_compare $cpio_version "2.11"; then + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time --directory=$rpm_name + ret=$? + else + current_dir="$PWD" + cd $rpm_name + rpm2cpio $debuginfo_rpm | cpio --quiet --extract --make-directories --preserve-modification-time + ret=$? + cd $current_dir + fi + if [ $ret -eq 1 ]; then + echo "failed to extract rpm $debuginfo_rpm to $PWD/$rpm_name directory" + rm -rf $rpm_name + exit 1 + fi debuginfo_path="$PWD/$rpm_name/usr/lib/debug" debuginfo_extension=".debug" else @@ -76,15 +96,16 @@ else fi # NOTE: backtrace file should contain only the lines which need to be resolved -for bt in $(grep glusterfs $backtrace_file) +for bt in $(cat $backtrace_file) do libname=$(echo $bt | cut -f 1 -d '(') addr=$(echo $bt | cut -f 2 -d '(' | cut -f 1 -d ')') - # only unresolved addresses start with a '+' - if echo $addr | egrep '^\+' >/dev/null 2>&1 ; then - newbt=( $(eu-addr2line --functions --exe=${debuginfo_path}${libname}${debuginfo_extension} $addr) ) - echo "$bt ${newbt[*]}" + libpath=${debuginfo_path}${libname}${debuginfo_extension} + if [ ! -f $libpath ]; then + continue fi + newbt=( $(eu-addr2line --functions --exe=$libpath $addr) ) + echo "$bt ${newbt[*]}" done # remove the temporary directory diff --git a/extras/devel-tools/strace-brick.sh b/extras/devel-tools/strace-brick.sh new file mode 100755 index 00000000000..a140729111c --- /dev/null +++ b/extras/devel-tools/strace-brick.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Usage: +# nice -n -19 strace-brick.sh glusterfsd 50 + +brick_process_name=$1 +min_watch_cpu=$2 +if [ ! $brick_process_name ]; then + brick_process_name=glusterfsd +fi + +if [ ! $min_watch_cpu ]; then + min_watch_cpu=50 +fi + +echo "min_watch_cpu: $min_watch_cpu" + +break=false + +while ! $break; +do + mypids=( $(pgrep $brick_process_name) ) + echo "mypids: ${mypids[*]}" + + pid_args=$(echo ${mypids[*]} | sed -e 's/ / -p /g;s/^/-p /') + echo "pid_args: $pid_args" + + pcpu=( $(ps $pid_args -o pcpu -h ) ) + echo "pcpu: ${pcpu[*]}" + + wait_longer=false + + for i in $( seq 0 $((${#pcpu[*]} - 1)) ) + do + echo "i: $i" + echo "mypids[$i]: ${mypids[$i]}" + + int_pcpu=$(echo ${pcpu[$i]} | cut -f 1 -d '.') + echo "int_pcpu: $int_pcpu" + if [ ! $int_pcpu ] || [ ! $min_watch_cpu ]; then + break=true + echo "breaking" + fi + if [ $int_pcpu -ge $min_watch_cpu ]; then + wait_longer=true + mydirname="${brick_process_name}-${mypids[$i]}-$(date --utc +'%Y%m%d-%H%M%S.%N')" + $(mkdir $mydirname && cd $mydirname && timeout --kill-after=5 --signal=KILL 60 nice -n -19 strace -p ${mypids[$i]} -ff -tt -T -o $brick_process_name) & + fi + done + + if $wait_longer; then + sleep 90 + else + sleep 15 + fi +done diff --git a/extras/distributed-testing/README b/extras/distributed-testing/README new file mode 100644 index 00000000000..928d943f211 --- /dev/null +++ b/extras/distributed-testing/README @@ -0,0 +1,28 @@ +PROBLEM + +The testing methodology of Gluster is extremely slow. It takes a very long time (6+ hrs) to run the basic tests on a single machine. It takes about 20+ hours to run code analysis version of tests like valgrind, asan, tsan etc. + +SOLUTION + +The fundamental problem is that the tests cannot be parallelized on a single machine. The natural solution is to run these tests on a cluster of machines. In a nutshell, apply map-reduce to run unit tests. + +WORK @ Facebook + +At Facebook we have applied the map-reduce approach to testing and have observed 10X improvements. + +The solution supports the following + +Distribute tests across machines, collect results/logs +Share worker pool across different testers +Try failure 3 times on 3 different machines before calling it a failure +Support running asan, valgrind, asan-noleaks +Self management of worker pools. The clients will manage the worker pool including version update, no manual maintenance required +WORK + +Port the code from gluster-fb-3.8 to gluster master + +HOW TO RUN + +./extras/distributed-testing/distributed-test.sh --hosts '<h1> <h2> <h3>' + +All hosts should have no password for ssh via root. This can be achieved with keys setup on the client and the server machines. diff --git a/extras/distributed-testing/distributed-test-build-env b/extras/distributed-testing/distributed-test-build-env new file mode 100644 index 00000000000..cd68ff717da --- /dev/null +++ b/extras/distributed-testing/distributed-test-build-env @@ -0,0 +1,20 @@ +#!/bin/bash + +GF_CONF_OPTS="--localstatedir=/var --sysconfdir /var/lib --prefix /usr --libdir /usr/lib64 \ + --enable-bd-xlator=yes --enable-debug --enable-gnfs" + +if [ -x /usr/lib/rpm/redhat/dist.sh ]; then + REDHAT_MAJOR=$(/usr/lib/rpm/redhat/dist.sh --distnum) +else + REDHAT_MAJOR=0 +fi + +ASAN_ENABLED=${ASAN_ENABLED:=0} +if [ "$ASAN_ENABLED" -eq "1" ]; then + GF_CONF_OPTS="$GF_CONF_OPTS --with-asan" +fi + +GF_CONF_OPTS="$GF_CONF_OPTS --with-systemd" +export GF_CONF_OPTS + +export CFLAGS="-O0 -ggdb -fPIC -Wall" diff --git a/extras/distributed-testing/distributed-test-build.sh b/extras/distributed-testing/distributed-test-build.sh new file mode 100755 index 00000000000..e8910d8425c --- /dev/null +++ b/extras/distributed-testing/distributed-test-build.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +EXTRA_CONFIGURE_ARGS="$@" +ASAN_REQUESTED=false +for arg in $EXTRA_CONFIGURE_ARGS; do + if [ $arg == "--with-asan" ]; then + echo "Requested ASAN, cleaning build first." + make -j distclean || true + touch .with_asan + ASAN_REQUESTED=true + fi +done + +if [ $ASAN_REQUESTED == false ]; then + if [ -f .with_asan ]; then + echo "Previous build was with ASAN, cleaning build first." + make -j distclean || true + rm -v .with_asan + fi +fi + +source extras/distributed-testing/distributed-test-build-env +./autogen.sh +./configure $GF_CONF_OPTS $EXTRA_CONFIGURE_ARGS +make -j diff --git a/extras/distributed-testing/distributed-test-env b/extras/distributed-testing/distributed-test-env new file mode 100644 index 00000000000..36fdd82e5dd --- /dev/null +++ b/extras/distributed-testing/distributed-test-env @@ -0,0 +1,48 @@ +#!/bin/bash + +SMOKE_TESTS="\ + tests/basic/*.t\ + tests/basic/afr/*.t\ + tests/basic/distribute/*.t\ + tests/bugs/fb*.t\ + tests/features/brick-min-free-space.t\ +" + +KNOWN_FLAKY_TESTS="\ +" + +BROKEN_TESTS="\ + tests/features/lock_revocation.t\ + tests/features/recon.t\ + tests/features/fdl-overflow.t\ + tests/features/fdl.t\ + tests/features/ipc.t\ + tests/bugs/distribute/bug-1247563.t\ + tests/bugs/distribute/bug-1543279.t\ + tests/bugs/distribute/bug-1066798.t\ + tests/bugs/ec/bug-1304988.t\ + tests/bugs/unclassified/bug-1357397.t\ + tests/bugs/quota/bug-1235182.t\ + tests/bugs/fuse/bug-1309462.t\ + tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\ + tests/bugs/stripe/bug-1002207.t\ + tests/bugs/stripe/bug-1111454.t\ + tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\ + tests/bugs/write-behind/bug-1279730.t\ + tests/bugs/gfapi/bug-1093594.t\ + tests/bugs/replicate/bug-1473026.t\ + tests/bugs/replicate/bug-802417.t\ + tests/basic/inode-leak.t\ + tests/basic/distribute/force-migration.t\ + tests/basic/ec/heal-info.t\ + tests/basic/ec/ec-seek.t\ + tests/basic/jbr/jbr-volgen.t\ + tests/basic/jbr/jbr.t\ + tests/basic/afr/tarissue.t\ + tests/basic/tier/tierd_check.t\ + tests/basic/gfapi/bug1291259.t\ +" + +SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ') +KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ') +BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ') diff --git a/extras/distributed-testing/distributed-test-runner.py b/extras/distributed-testing/distributed-test-runner.py new file mode 100755 index 00000000000..5a07e2feab1 --- /dev/null +++ b/extras/distributed-testing/distributed-test-runner.py @@ -0,0 +1,859 @@ +#!/usr/bin/python3 + +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals +from __future__ import print_function +import re +import sys +import fcntl +import base64 +import threading +import socket +import os +import shlex +import argparse +import subprocess +import time +import SimpleXMLRPCServer +import xmlrpclib +import md5 +import httplib +import uuid + +DEFAULT_PORT = 9999 +TEST_TIMEOUT_S = 15 * 60 +CLIENT_CONNECT_TIMEOUT_S = 10 +CLIENT_TIMEOUT_S = 60 +PATCH_FILE_UID = str(uuid.uuid4()) +SSH_TIMEOUT_S = 10 +MAX_ATTEMPTS = 3 +ADDRESS_FAMILY = 'IPv4' + + +def socket_instance(address_family): + if address_family.upper() == 'ipv4'.upper(): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + elif address_family.upper() == 'ipv6'.upper(): + return socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + else: + Log.error("Invalid IP address family") + sys.exit(1) + + +def patch_file(): + return "/tmp/%s-patch.tar.gz" % PATCH_FILE_UID + +# .............................................................................. +# SimpleXMLRPCServer IPvX Wrapper +# .............................................................................. + + +class GeneralXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): + def __init__(self, addr): + SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr) + + def server_bind(self): + if self.socket: + self.socket.close() + self.socket = socket_instance(args.address_family) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + SimpleXMLRPCServer.SimpleXMLRPCServer.server_bind(self) + + +class HTTPConnection(httplib.HTTPConnection): + def __init__(self, host): + self.host = host + httplib.HTTPConnection.__init__(self, host) + + def connect(self): + old_timeout = socket.getdefaulttimeout() + self.sock = socket.create_connection((self.host, self.port), + timeout=CLIENT_CONNECT_TIMEOUT_S) + self.sock.settimeout(old_timeout) + + +class IPTransport(xmlrpclib.Transport): + def __init__(self, *args, **kwargs): + xmlrpclib.Transport.__init__(self, *args, **kwargs) + + def make_connection(self, host): + return HTTPConnection(host) + + +# .............................................................................. +# Common +# .............................................................................. + + +class Timer: + def __init__(self): + self.start = time.time() + + def elapsed_s(self): + return int(time.time() - self.start) + + def reset(self): + ret = self.elapsed_s() + self.start = time.time() + return ret + + +def encode(buf): + return base64.b16encode(buf) + + +def decode(buf): + return base64.b16decode(buf) + + +def get_file_content(path): + with open(path, "r") as f: + return f.read() + + +def write_to_file(path, data): + with open(path, "w") as f: + f.write(data) + + +def failsafe(fn, args=()): + try: + return (True, fn(*args)) + except (xmlrpclib.Fault, xmlrpclib.ProtocolError, xmlrpclib.ResponseError, + Exception) as err: + Log.debug(str(err)) + return (False, None) + + +class LogLevel: + DEBUG = 2 + ERROR = 1 + CLI = 0 + + +class Log: + LOGLEVEL = LogLevel.ERROR + + @staticmethod + def _normalize(msg): + return msg[:100] + + @staticmethod + def debug(msg): + if Log.LOGLEVEL >= LogLevel.DEBUG: + sys.stdout.write("<debug> %s\n" % Log._normalize(msg)) + sys.stdout.flush() + + @staticmethod + def error(msg): + sys.stderr.write("<error> %s\n" % Log._normalize(msg)) + + @staticmethod + def header(msg): + sys.stderr.write("* %s *\n" % Log._normalize(msg)) + + @staticmethod + def cli(msg): + sys.stderr.write("%s\n" % msg) + + +class Shell: + def __init__(self, cwd=None, logpath=None): + self.cwd = cwd + self.shell = True + self.redirect = open(os.devnull if not logpath else logpath, "wr+") + + def __del__(self): + self.redirect.close() + + def cd(self, cwd): + Log.debug("cd %s" % cwd) + self.cwd = cwd + + def truncate(self): + self.redirect.truncate(0) + + def read_logs(self): + self.redirect.seek(0) + return self.redirect.read() + + def check_call(self, cmd): + status = self.call(cmd) + if status: + raise Exception("Error running command %s. status=%s" + % (cmd, status)) + + def call(self, cmd): + if isinstance(cmd, list): + return self._calls(cmd) + + return self._call(cmd) + + def ssh(self, hostname, cmd, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s ssh %s root@%s \"%s\"" % + (SSH_TIMEOUT_S, flags, hostname, cmd)) + + def scp(self, hostname, src, dest, id_rsa=None): + flags = "" if not id_rsa else "-i " + id_rsa + return self.call("timeout %s scp %s %s root@%s:%s" % + (SSH_TIMEOUT_S, flags, src, hostname, dest)) + + def output(self, cmd, cwd=None): + Log.debug("%s> %s" % (cwd, cmd)) + return subprocess.check_output(shlex.split(cmd), cwd=self.cwd) + + def _calls(self, cmds): + Log.debug("Running commands. %s" % cmds) + for c in cmds: + status = self.call(c) + if status: + Log.error("Commands failed with %s" % status) + return status + return 0 + + def _call(self, cmd): + if not self.shell: + cmd = shlex.split(cmd) + + Log.debug("%s> %s" % (self.cwd, cmd)) + + status = subprocess.call(cmd, cwd=self.cwd, shell=self.shell, + stdout=self.redirect, stderr=self.redirect) + + Log.debug("return %s" % status) + return status + + +# .............................................................................. +# Server role +# .............................................................................. + +class TestServer: + def __init__(self, port, scratchdir): + self.port = port + self.scratchdir = scratchdir + self.shell = Shell() + self.rpc = None + self.pidf = None + + self.shell.check_call("mkdir -p %s" % self.scratchdir) + self._process_lock() + + def __del__(self): + if self.pidf: + self.pidf.close() + + def init(self): + Log.debug("Starting xmlrpc server on port %s" % self.port) + self.rpc = GeneralXMLRPCServer(("", self.port)) + self.rpc.register_instance(Handlers(self.scratchdir)) + + def serve(self): + (status, _) = failsafe(self.rpc.serve_forever) + Log.cli("== End ==") + + def _process_lock(self): + pid_filename = os.path.basename(__file__).replace("/", "-") + pid_filepath = "%s/%s.pid" % (self.scratchdir, pid_filename) + self.pidf = open(pid_filepath, "w") + try: + fcntl.lockf(self.pidf, fcntl.LOCK_EX | fcntl.LOCK_NB) + # We have the lock, kick anybody listening on this port + self.shell.call("kill $(lsof -t -i:%s)" % self.port) + except IOError: + Log.error("Another process instance is running") + sys.exit(0) + +# +# Server Handler +# + + +handler_lock = threading.Lock() +handler_serving_since = Timer() + + +def synchronized(func): + def decorator(*args, **kws): + handler_lock.acquire() + h = args[0] + try: + h.shell.truncate() + ret = func(*args, **kws) + return ret + except Exception() as err: + Log.error(str(err)) + Log.error(decode(h._log_content())) + raise + finally: + handler_lock.release() + handler_serving_since.reset() + + return decorator + + +class Handlers: + def __init__(self, scratchdir): + self.client_id = None + self.scratchdir = scratchdir + self.gluster_root = "%s/glusterfs" % self.scratchdir + self.shell = Shell(logpath="%s/test-handlers.log" % self.scratchdir) + + def hello(self, id): + if not handler_lock.acquire(False): + return False + try: + return self._hello_locked(id) + finally: + handler_lock.release() + + def _hello_locked(self, id): + if handler_serving_since.elapsed_s() > CLIENT_TIMEOUT_S: + Log.debug("Disconnected client %s" % self.client_id) + self.client_id = None + + if not self.client_id: + self.client_id = id + handler_serving_since.reset() + return True + + return (id == self.client_id) + + @synchronized + def ping(self, id=None): + if id: + return id == self.client_id + return True + + @synchronized + def bye(self, id): + assert id == self.client_id + self.client_id = None + handler_serving_since.reset() + return True + + @synchronized + def cleanup(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.check_call("PATH=.:$PATH; sudo ./clean_gfs_devserver.sh") + return True + + @synchronized + def copy(self, id, name, content): + with open("%s/%s" % (self.scratchdir, name), "w+") as f: + f.write(decode(content)) + return True + + @synchronized + def copygzip(self, id, content): + assert id == self.client_id + gzipfile = "%s/tmp.tar.gz" % self.scratchdir + tarfile = "%s/tmp.tar" % self.scratchdir + self.shell.check_call("rm -f %s" % gzipfile) + self.shell.check_call("rm -f %s" % tarfile) + write_to_file(gzipfile, decode(content)) + + self.shell.cd(self.scratchdir) + self.shell.check_call("rm -r -f %s" % self.gluster_root) + self.shell.check_call("mkdir -p %s" % self.gluster_root) + + self.shell.cd(self.gluster_root) + cmds = [ + "gunzip -f -q %s" % gzipfile, + "tar -xvf %s" % tarfile + ] + return self.shell.call(cmds) == 0 + + @synchronized + def build(self, id, asan=False): + assert id == self.client_id + self.shell.cd(self.gluster_root) + self.shell.call("make clean") + env = "ASAN_ENABLED=1" if asan else "" + return self.shell.call( + "%s ./extras/distributed-testing/distributed-test-build.sh" % env) == 0 + + @synchronized + def install(self, id): + assert id == self.client_id + self.shell.cd(self.gluster_root) + return self.shell.call("make install") == 0 + + @synchronized + def prove(self, id, test, timeout, valgrind="no", asan_noleaks=True): + assert id == self.client_id + self.shell.cd(self.gluster_root) + env = "DEBUG=1 " + if valgrind == "memcheck" or valgrind == "yes": + cmd = "valgrind" + cmd += " --tool=memcheck --leak-check=full --track-origins=yes" + cmd += " --show-leak-kinds=all -v prove -v" + elif valgrind == "drd": + cmd = "valgrind" + cmd += " --tool=drd -v prove -v" + elif asan_noleaks: + cmd = "prove -v" + env += "ASAN_OPTIONS=detect_leaks=0 " + else: + cmd = "prove -v" + + status = self.shell.call( + "%s timeout %s %s %s" % (env, timeout, cmd, test)) + + if status != 0: + return (False, self._log_content()) + return (True, "") + + def _log_content(self): + return encode(self.shell.read_logs()) + +# .............................................................................. +# Cli role +# .............................................................................. + + +class RPCConnection((threading.Thread)): + def __init__(self, host, port, path, cb): + threading.Thread.__init__(self) + self.host = host + self.port = port + self.path = path + self.shell = Shell() + self.cb = cb + self.stop = False + self.proxy = None + self.logid = "%s:%s" % (self.host, self.port) + + def connect(self): + (status, ret) = failsafe(self._connect) + return (status and ret) + + def _connect(self): + url = "http://%s:%s" % (self.host, self.port) + self.proxy = xmlrpclib.ServerProxy(url, transport=IPTransport()) + return self.proxy.hello(self.cb.id) + + def disconnect(self): + self.stop = True + + def ping(self): + return self.proxy.ping() + + def init(self): + return self._copy() and self._compile_and_install() + + def run(self): + (status, ret) = failsafe(self.init) + if not status: + self.cb.note_lost_connection(self) + return + elif not ret: + self.cb.note_setup_failed(self) + return + + while not self.stop: + (status, ret) = failsafe(self._run) + if not status or not ret: + self.cb.note_lost_connection(self) + break + time.sleep(0) + + failsafe(self.proxy.bye, (self.cb.id,)) + Log.debug("%s connection thread stopped" % self.host) + + def _run(self): + test = self.cb.next_test() + (status, _) = failsafe(self._execute_next, (test,)) + if not status: + self.cb.note_retry(test) + return False + return True + + def _execute_next(self, test): + if not test: + time.sleep(1) + return + + (status, error) = self.proxy.prove(self.cb.id, test, + self.cb.test_timeout, + self.cb.valgrind, + self.cb.asan_noleaks) + if status: + self.cb.note_done(test) + else: + self.cb.note_error(test, error) + + def _compile_and_install(self): + Log.debug("<%s> Build " % self.logid) + asan = self.cb.asan or self.cb.asan_noleaks + return (self.proxy.build(self.cb.id, asan) and + self.proxy.install(self.cb.id)) + + def _copy(self): + return self._copy_gzip() + + def _copy_gzip(self): + Log.cli("<%s> copying and compiling %s to remote" % + (self.logid, self.path)) + data = encode(get_file_content(patch_file())) + Log.debug("GZIP size = %s B" % len(data)) + return self.proxy.copygzip(self.cb.id, data) + + +class RPCConnectionPool: + def __init__(self, gluster_path, hosts, n, id_rsa): + self.gluster_path = gluster_path + self.hosts = hosts + self.conns = [] + self.faulty = [] + self.n = int(len(hosts) / 2) + 1 if not n else n + self.id_rsa = id_rsa + self.stop = False + self.scanner = threading.Thread(target=self._scan_hosts_loop) + self.kicker = threading.Thread(target=self._kick_hosts_loop) + self.shell = Shell() + self.since_start = Timer() + + self.shell.check_call("rm -f %s" % patch_file()) + self.shell.check_call("tar -zcvf %s ." % patch_file()) + self.id = md5.new(get_file_content(patch_file())).hexdigest() + Log.cli("client UID %s" % self.id) + Log.cli("patch UID %s" % PATCH_FILE_UID) + + def __del__(self): + self.shell.check_call("rm -f %s" % patch_file()) + + def pool_status(self): + elapsed_m = int(self.since_start.elapsed_s() / 60) + return "%s/%s connected, %smin elapsed" % (len(self.conns), self.n, + elapsed_m) + + def connect(self): + Log.debug("Starting scanner") + self.scanner.start() + self.kicker.start() + + def disconnect(self): + self.stop = True + for conn in self.conns: + conn.disconnect() + + def note_lost_connection(self, conn): + Log.cli("lost connection to %s" % conn.host) + self.conns.remove(conn) + self.hosts.append((conn.host, conn.port)) + + def note_setup_failed(self, conn): + Log.error("Setup failed on %s:%s" % (conn.host, conn.port)) + self.conns.remove(conn) + self.faulty.append((conn.host, conn.port)) + + def _scan_hosts_loop(self): + Log.debug("Scanner thread started") + while not self.stop: + failsafe(self._scan_hosts) + time.sleep(5) + + def _scan_hosts(self): + if len(self.hosts) == 0 and len(self.conns) == 0: + Log.error("no more hosts available to loadbalance") + sys.exit(1) + + for (host, port) in self.hosts: + if (len(self.conns) >= self.n) or self.stop: + break + self._scan_host(host, port) + + def _scan_host(self, host, port): + Log.debug("scanning %s:%s" % (host, port)) + c = RPCConnection(host, port, self.gluster_path, self) + (status, result) = failsafe(c.connect) + if status and result: + self.hosts.remove((host, port)) + Log.debug("Connected to %s:%s" % (host, port)) + self.conns.append(c) + c.start() + Log.debug("%s / %s connected" % (len(self.conns), self.n)) + else: + Log.debug("Failed to connect to %s:%s" % (host, port)) + + def _kick_hosts_loop(self): + Log.debug("Kick thread started") + while not self.stop: + time.sleep(10) + failsafe(self._kick_hosts) + + Log.debug("Kick thread stopped") + + def _is_pingable(self, host, port): + c = RPCConnection(host, port, self.gluster_path, self) + failsafe(c.connect) + (status, result) = failsafe(c.ping) + return status and result + + def _kick_hosts(self): + # Do not kick hosts if we have the optimal number of connections + if (len(self.conns) >= self.n) or self.stop: + Log.debug("Skip kicking hosts") + return + + # Check and if dead kick all hosts + for (host, port) in self.hosts: + if self.stop: + Log.debug("Break kicking hosts") + break + + if self._is_pingable(host, port): + Log.debug("Host=%s is alive. Won't kick" % host) + continue + + Log.debug("Kicking %s" % host) + mypath = sys.argv[0] + myname = os.path.basename(mypath) + destpath = "/tmp/%s" % myname + sh = Shell() + sh.scp(host, mypath, destpath, self.id_rsa) + sh.ssh(host, "nohup %s --server &>> %s.log &" % + (destpath, destpath), self.id_rsa) + + def join(self): + self.scanner.join() + self.kicker.join() + for c in self.conns: + c.join() + + +# .............................................................................. +# test role +# .............................................................................. + +class TestRunner(RPCConnectionPool): + def __init__(self, gluster_path, hosts, n, tests, flaky_tests, valgrind, + asan, asan_noleaks, id_rsa, test_timeout): + RPCConnectionPool.__init__(self, gluster_path, self._parse_hosts(hosts), + n, id_rsa) + self.flaky_tests = flaky_tests.split(" ") + self.pending = [] + self.done = [] + self.error = [] + self.retry = {} + self.error_logs = [] + self.stats_timer = Timer() + self.valgrind = valgrind + self.asan = asan + self.asan_noleaks = asan_noleaks + self.test_timeout = test_timeout + + self.tests = self._get_tests(tests) + + Log.debug("tests: %s" % self.tests) + + def _get_tests(self, tests): + if not tests or tests == "all": + return self._not_flaky(self._all()) + elif tests == "flaky": + return self.flaky_tests + else: + return self._not_flaky(tests.strip().split(" ")) + + def run(self): + self.connect() + self.join() + return len(self.error) + + def _pretty_print(self, data): + if isinstance(data, list): + str = "" + for i in data: + str = "%s %s" % (str, i) + return str + return "%s" % data + + def print_result(self): + Log.cli("== RESULTS ==") + Log.cli("SUCCESS : %s" % len(self.done)) + Log.cli("ERRORS : %s" % len(self.error)) + Log.cli("== ERRORS ==") + Log.cli(self._pretty_print(self.error)) + Log.cli("== LOGS ==") + Log.cli(self._pretty_print(self.error_logs)) + Log.cli("== END ==") + + def next_test(self): + if len(self.tests): + test = self.tests.pop() + self.pending.append(test) + return test + + if not len(self.pending): + self.disconnect() + + return None + + def _pct_completed(self): + total = len(self.tests) + len(self.pending) + len(self.done) + total += len(self.error) + completed = len(self.done) + len(self.error) + return 0 if not total else int(completed / total * 100) + + def note_done(self, test): + Log.cli("%s PASS (%s%% done) (%s)" % (test, self._pct_completed(), + self.pool_status())) + self.pending.remove(test) + self.done.append(test) + if test in self.retry: + del self.retry[test] + + def note_error(self, test, errstr): + Log.cli("%s FAIL" % test) + self.pending.remove(test) + if test not in self.retry: + self.retry[test] = 1 + + if errstr: + path = "%s/%s-%s.log" % ("/tmp", test.replace("/", "-"), + self.retry[test]) + failsafe(write_to_file, (path, decode(errstr),)) + self.error_logs.append(path) + + if self.retry[test] < MAX_ATTEMPTS: + self.retry[test] += 1 + Log.debug("retry test %s attempt %s" % (test, self.retry[test])) + self.tests.append(test) + else: + Log.debug("giveup attempt test %s" % test) + del self.retry[test] + self.error.append(test) + + def note_retry(self, test): + Log.cli("retry %s on another host" % test) + self.pending.remove(test) + self.tests.append(test) + + # + # test classifications + # + def _all(self): + return self._list_tests(["tests"], recursive=True) + + def _not_flaky(self, tests): + for t in self.flaky_tests: + if t in tests: + tests.remove(t) + return tests + + def _list_tests(self, prefixes, recursive=False, ignore_ifnotexist=False): + tests = [] + for prefix in prefixes: + real_path = "%s/%s" % (self.gluster_path, prefix) + if not os.path.exists(real_path) and ignore_ifnotexist: + continue + for f in os.listdir(real_path): + if os.path.isdir(real_path + "/" + f): + if recursive: + tests += self._list_tests([prefix + "/" + f], recursive) + else: + if re.match(r".*\.t$", f): + tests += [prefix + "/" + f] + return tests + + def _parse_hosts(self, hosts): + ret = [] + for h in args.hosts.split(" "): + ret.append((h, DEFAULT_PORT)) + Log.debug(ret) + return ret + +# .............................................................................. +# Roles entry point +# .............................................................................. + + +def run_as_server(args): + if not args.server_path: + Log.error("please provide server path") + return 1 + + server = TestServer(args.port, args.server_path) + server.init() + server.serve() + return 0 + + +def run_as_tester(args): + Log.header("GLUSTER TEST CLI") + + Log.debug("args = %s" % args) + + tests = TestRunner(args.gluster_path, args.hosts, args.n, args.tests, + args.flaky_tests, valgrind=args.valgrind, + asan=args.asan, asan_noleaks=args.asan_noleaks, + id_rsa=args.id_rsa, test_timeout=args.test_timeout) + result = tests.run() + tests.print_result() + return result + +# .............................................................................. +# main +# .............................................................................. + + +def main(args): + if args.v: + Log.LOGLEVEL = LogLevel.DEBUG + + if args.server and args.tester: + Log.error("Invalid arguments. More than one role specified") + sys.exit(1) + + if args.server: + sys.exit(run_as_server(args)) + elif args.tester: + sys.exit(run_as_tester(args)) + else: + Log.error("please specify a mode for CI") + parser.print_help() + sys.exit(1) + + +parser = argparse.ArgumentParser(description="Gluster CI") + +# server role +parser.add_argument("--server", help="start server", action="store_true") +parser.add_argument("--server_path", help="server scratch space", + default="/tmp/gluster-test") +parser.add_argument("--host", help="server address to listen", default="") +parser.add_argument("--port", help="server port to listen", + type=int, default=DEFAULT_PORT) +# test role +parser.add_argument("--tester", help="start tester", action="store_true") +parser.add_argument("--valgrind[=memcheck,drd]", + help="run tests with valgrind tool 'memcheck' or 'drd'", + default="no") +parser.add_argument("--asan", help="test with asan enabled", + action="store_true") +parser.add_argument("--asan-noleaks", help="test with asan but no mem leaks", + action="store_true") +parser.add_argument("--tests", help="all/flaky/list of tests", default=None) +parser.add_argument("--flaky_tests", help="list of flaky tests", default=None) +parser.add_argument("--n", help="max number of machines to use", type=int, + default=0) +parser.add_argument("--hosts", help="list of worker machines") +parser.add_argument("--gluster_path", help="gluster path to test", + default=os.getcwd()) +parser.add_argument("--id-rsa", help="private key to use for ssh", + default=None) +parser.add_argument("--test-timeout", + help="test timeout in sec (default 15min)", + default=TEST_TIMEOUT_S) +# general +parser.add_argument("-v", help="verbose", action="store_true") +parser.add_argument("--address_family", help="IPv6 or IPv4 to use", + default=ADDRESS_FAMILY) + +args = parser.parse_args() + +main(args) diff --git a/extras/distributed-testing/distributed-test.sh b/extras/distributed-testing/distributed-test.sh new file mode 100755 index 00000000000..8f1e0310f33 --- /dev/null +++ b/extras/distributed-testing/distributed-test.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +source ./extras/distributed-testing/distributed-test-env + +N=0 +TESTS='all' +FLAKY=$KNOWN_FLAKY_TESTS +BROKEN=$BROKEN_TESTS +TEST_TIMEOUT_S=900 +ADDRESS_FAMILY='IPv4' + +FLAGS="" + +function print_env { + echo "Settings:" + echo "N=$N" + echo -e "-------\nHOSTS\n$HOSTS\n-------" + echo -e "TESTS\n$TESTS\n-------" + echo -e "SKIP\n$FLAKY $BROKEN\n-------" + echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n" +} + +function cleanup { + rm -f /tmp/test*.log +} + +function usage { + echo "Usage: $0 [-h or --help] [-v or --verbose] + [--all] [--flaky] [--smoke] [--broken] + [--valgrind] [--asan] [--asan-noleaks] + [--hosts <hosts>] [-n <parallelism>] + [--tests <tests>] + [--id-rsa <ssh private key>] + [--address_family <IPv4 or IPv6>] + " +} + +function parse_args () { + args=`getopt \ + -o hvn: \ + --long help,verbose,address_family:,valgrind,asan,asan-noleaks,all,\ +smoke,flaky,broken,hosts:,tests:,id-rsa:,test-timeout: \ + -n 'fb-remote-test.sh' -- "$@"` + + if [ $? != 0 ]; then + echo "Error parsing getopt" + exit 1 + fi + + eval set -- "$args" + + while true; do + case "$1" in + -h | --help) usage ; exit 1 ;; + -v | --verbose) FLAGS="$FLAGS -v" ; shift ;; + --address_family) ADDRESS_FAMILY=$2; shift 2 ;; + --valgrind) FLAGS="$FLAGS --valgrind" ; shift ;; + --asan-noleaks) FLAGS="$FLAGS --asan-noleaks"; shift ;; + --asan) FLAGS="$FLAGS --asan" ; shift ;; + --hosts) HOSTS=$2; shift 2 ;; + --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;; + --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;; + --all) TESTS='all' ; shift 1 ;; + --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;; + --smoke) TESTS=$SMOKE_TESTS; shift 1 ;; + --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;; + --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;; + -n) N=$2; shift 2 ;; + *) break ;; + esac + done + run_tests_args="$@" +} + +function main { + parse_args "$@" + + if [ -z "$HOSTS" ]; then + echo "Please provide hosts to run the tests in" + exit -1 + fi + + print_env + + cleanup + + "extras/distributed-testing/distributed-test-runner.py" $FLAGS --tester \ + --n "$N" --hosts "$HOSTS" --tests "$TESTS" \ + --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S" \ + --address_family "$ADDRESS_FAMILY" + + exit $? +} + +main "$@" diff --git a/extras/ec-heal-script/README.md b/extras/ec-heal-script/README.md new file mode 100644 index 00000000000..aaefd6681f6 --- /dev/null +++ b/extras/ec-heal-script/README.md @@ -0,0 +1,69 @@ +# gluster-heal-scripts +Scripts to correct extended attributes of fragments of files to make them healble. + +Following are the guidelines/suggestions to use these scripts. + +1 - Passwordless ssh should be setup for all the nodes of the cluster. + +2 - Scripts should be executed from one of these nodes. + +3 - Make sure NO "IO" is going on for the files for which we are running +these two scripts. + +4 - There should be no heal going on for the file for which xattrs are being +set by correct_pending_heals.sh. Disable the self heal while running this script. + +5 - All the bricks of the volume should be UP to identify good and bad fragments +and to decide if an entry is healble or not. + +6 - If correct_pending_heals.sh is stopped in the middle while it was processing +healble entries, it is suggested to re-run gfid_needing_heal_parallel.sh to create +latest list of healble and non healble entries and "potential_heal" "can_not_heal" files. + +7 - Based on the number of entries, these files might take time to get and set the +stats and xattrs of entries. + +8 - A backup of the fragments will be taken on <brick path>/.glusterfs/correct_pending_heals + directory with a file name same as gfid. + +9 - Once the correctness of the file gets verified by user, these backup should be removed. + +10 - Make sure we have enough space on bricks to take these backups. + +11 - At the end this will create two files - + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +12 - It is suggested that the integrity of the data of files, which were modified and healed, + should be checked by the user. + + +Usage: + +Following are the sequence of steps to use these scripts - + +1 - ./gfid_needing_heal_parallel.sh <volume name> + + Execute gfid_needing_heal_parallel.sh with volume name to create list of files which could + be healed and can not be healed. It creates "potential_heal" and "can_not_heal" files. + During execution, it also displays the list of files on consol with the verdict. + +2 - ./correct_pending_heals.sh + + Execute correct_pending_heals.sh without any argument. This script processes entries present + in "heal" file. It asks user to enter how many files we want to process in one attempt. + Once the count is provided, this script will fetch the entries one by one from "potential_heal" file and takes necessary action. + If at this point also a file can not be healed, it will be pushed to "can_not_heal" file. + If a file can be healed, this script will modify the xattrs of that file fragments and create an entry in "modified_and_backedup_files" file + +3 - At the end, all the entries of "potential_heal" will be processed and based on the processing only two files will be left. + + 1 - modified_and_backedup_files - Contains list of files which have been modified and should be healed. + 2 - can_not_heal - Contains list of files which can not be healed. + +Logs and other files - + +1 - modified_and_backedup_files - It contains all the files which could be healed and the location of backup of each fragments. +2 - can_not_heal - It contains all the files which can not be healed. +3 - potential_heal - List of files which could be healed and should be processed by "correct_pending_heals.sh" +4 - /var/log/glusterfs/ec-heal-script.log - It contains logs of both the files. diff --git a/extras/ec-heal-script/correct_pending_heals.sh b/extras/ec-heal-script/correct_pending_heals.sh new file mode 100755 index 00000000000..c9f19dd7c89 --- /dev/null +++ b/extras/ec-heal-script/correct_pending_heals.sh @@ -0,0 +1,415 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script finally resets the xattrs of all the fragments of a file +# which can be healed as per gfid_needing_heal_parallel.sh. +# gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +# This script takes potential_heal as input and resets xattrs of all the fragments +# of those files present in this file and which could be healed as per +# trusted.ec.size xattar of the file else it will place the entry in can_not_heal +# file. Those entries which must be healed will be place in must_heal file +# after setting xattrs so that user can track those files. + + +MOD_BACKUP_FILES="modified_and_backedup_files" +CAN_NOT_HEAL="can_not_heal" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" +LINE_SEP="===================================================" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function desc () +{ + echo "" + echo "This script finally resets the xattrs of all the fragments of a file +which can be healed as per gfid_needing_heal_parallel.sh. +gfid_needing_heal_parallel.sh will produce two files, potential_heal and can_not_heal. +This script takes potential_heal as input and resets xattrs of all the fragments +of those files present in this file and which could be healed as per +trusted.ec.size xattar of the file else it will place the entry in can_not_heal +file. Those entries which must be healed will be place in must_heal file +after setting xattrs so that user can track those files." +} + +function _init () +{ + if [ $# -ne 0 ] + then + echo "usage: $0" + desc + exit 2 + fi + + if [ ! -f "potential_heal" ] + then + echo "Nothing to correct. File "potential_heal" does not exist" + echo "" + desc + exit 2 + fi +} + +function total_file_size_in_hex() +{ + local frag_size=$1 + local size=0 + local hex_size="" + + size=$((frag_size * 4)) + hex_size=$(printf '0x%016x' $size) + echo "$hex_size" +} + +function backup_file_fragment() +{ + local file_host=$1 + local file_entry=$2 + local gfid_actual_paths=$3 + local brick_root="" + local temp="" + local backup_dir="" + local cmd="" + local gfid="" + + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + temp=$(echo "$(basename "$BASH_SOURCE")" | cut -d '.' -f 1) + backup_dir=$(echo "${brick_root}/.glusterfs/${temp}") + file_entry=${file_entry//#} + + gfid=$(echo "${gfid_actual_paths}" | cut -d '|' -f 1 | cut -d '/' -f 5) + echo "${file_host}:${backup_dir}/${gfid}" >> "$MOD_BACKUP_FILES" + + cmd="mkdir -p ${backup_dir} && yes | cp -af ${file_entry} ${backup_dir}/${gfid} 2>/dev/null" + ssh -n "${file_host}" "${cmd}" +} + +function set_frag_xattr () +{ + local file_host=$1 + local file_entry=$2 + local good=$3 + local cmd1="" + local cmd2="" + local cmd="" + local version="0x00000000000000010000000000000001" + local dirty="0x00000000000000010000000000000001" + + if [[ $good -eq 0 ]] + then + version="0x00000000000000000000000000000000" + fi + + cmd1=" setfattr -n trusted.ec.version -v ${version} ${file_entry} &&" + cmd2=" setfattr -n trusted.ec.dirty -v ${dirty} ${file_entry}" + cmd=${cmd1}${cmd2} + ssh -n "${file_host}" "${cmd}" +} + +function set_version_dirty_xattr () +{ + local file_paths=$1 + local good=$2 + local gfid_actual_paths=$3 + local file_entry="" + local file_host="" + local bpath="" + + for bpath in ${file_paths//,/ } + do + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + backup_file_fragment "$file_host" "$file_entry" "$gfid_actual_paths" + file_entry=${file_entry//#} + set_frag_xattr "$file_host" "$file_entry" "$good" + done +} + +function match_size_xattr_quorum () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local size_xattr="" + local bpath="" + declare -A xattr_count + + for bpath in ${file_paths//,/ } + do + size_xattr="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + size_xattr=$(ssh -n "${file_host}" "${cmd}") + if [[ -n $size_xattr ]] + then + count=$((xattr_count["$size_xattr"] + 1)) + xattr_count["$size_xattr"]=${count} + if [[ $count -ge 4 ]] + then + echo "${size_xattr}" + return + fi + fi + done + echo "False" +} + +function match_version_xattr () +{ + local file_paths=$1 + local file_entry="" + local file_host="" + local cmd="" + local version="" + local bpath="" + declare -A ver_count + + for bpath in ${file_paths//,/ } + do + version="" + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + file_entry=${file_entry//#} + + cmd="getfattr -n trusted.ec.version -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.version" | cut -d '=' -f 2" + version=$(ssh -n "${file_host}" "${cmd}") + ver_count["$version"]=$((ver_count["$version"] + 1)) + done + for key in "${ver_count[@]}" + do + if [[ $key -ge 4 ]] + then + echo "True" + return + else + echo "False" + return + fi + done +} + +function match_stat_size_with_xattr () +{ + local bpath=$1 + local size=$2 + local file_stat=$3 + local xattr=$4 + local file_entry="" + local file_host="" + local cmd="" + local stat_output="" + local hex_size="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + + file_entry=${file_entry//#} + cmd="stat --format=%F:%B:%s $file_entry 2>/dev/null" + stat_output=$(ssh -n "${file_host}" "${cmd}") + echo "$stat_output" | grep -w "${file_stat}" > /dev/null + + if [[ $? -eq 0 ]] + then + cmd="getfattr -n trusted.ec.size -d -e hex ${file_entry} 2>/dev/null | grep -w "trusted.ec.size" | cut -d '=' -f 2" + hex_size=$(ssh -n "${file_host}" "${cmd}") + + if [[ -z $hex_size || "$hex_size" != "$xattr" ]] + then + echo "False" + return + fi + size_diff=$(printf '%d' $(( size - hex_size ))) + if [[ $size_diff -gt 2047 ]] + then + echo "False" + return + else + echo "True" + return + fi + else + echo "False" + return + fi +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function check_all_frag_and_set_xattr () +{ + local file_paths=$1 + local total_size=$2 + local file_stat=$3 + local bpath="" + local healthy_count=0 + local match="False" + local matching_bricks="" + local bad_bricks="" + local gfid_actual_paths="" + + for bpath in ${file_paths//,/ } + do + if [[ -n "$gfid_actual_paths" ]] + then + break + fi + gfid_actual_paths=$(find_file_paths "$bpath") + done + + match=$(match_size_xattr_quorum "$file_paths") + +# echo "${match} : $bpath" >> "$MOD_BACKUP_FILES" + + if [[ "$match" != "False" ]] + then + xattr="$match" + for bpath in ${file_paths//,/ } + do + match="False" + match=$(match_stat_size_with_xattr "$bpath" "$total_size" "$file_stat" "$xattr") + if [[ "$match" == "True" ]] + then + matching_bricks="${bpath},${matching_bricks}" + healthy_count=$((healthy_count + 1)) + else + bad_bricks="${bpath},${bad_bricks}" + fi + done + fi + + if [[ $healthy_count -ge 4 ]] + then + match="True" + echo "${LINE_SEP}" >> "$MOD_BACKUP_FILES" + echo "Modified : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$MOD_BACKUP_FILES" + set_version_dirty_xattr "$matching_bricks" 1 "$gfid_actual_paths" + set_version_dirty_xattr "$bad_bricks" 0 "$gfid_actual_paths" + else + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + + echo "$match" +} +function set_xattr() +{ + local count=$1 + local heal_entry="" + local file_stat="" + local frag_size="" + local total_size="" + local file_paths="" + local num="" + local can_heal_count=0 + + heal_log "Started $(basename $BASH_SOURCE) on $(date) " + + while read -r heal_entry + do + heal_log "$LINE_SEP" + heal_log "${heal_entry}" + + file_stat=$(echo "$heal_entry" | cut -d "|" -f 1) + frag_size=$(echo "$file_stat" | rev | cut -d ":" -f 1 | rev) + total_size="$(total_file_size_in_hex "$frag_size")" + file_paths=$(echo "$heal_entry" | cut -d "|" -f 2) + match=$(check_all_frag_and_set_xattr "$file_paths" "$total_size" "$file_stat") + if [[ "$match" == "True" ]] + then + can_heal_count=$((can_heal_count + 1)) + fi + + sed -i '1d' potential_heal + count=$((count - 1)) + if [ $count == 0 ] + then + num=$(cat potential_heal | wc -l) + heal_log "$LINE_SEP" + heal_log "${1} : Processed" + heal_log "${can_heal_count} : Modified to Heal" + heal_log "$((${1} - can_heal_count)) : Moved to can_not_heal." + heal_log "${num} : Pending as Potential Heal" + exit 0 + fi + + done < potential_heal +} + +function main () +{ + local count=0 + + read -p "Number of files to correct: [choose between 1-1000] (0 for All):" count + if [[ $count -lt 0 || $count -gt 1000 ]] + then + echo "Provide correct value:" + exit 2 + fi + + if [[ $count -eq 0 ]] + then + count=$(cat potential_heal | wc -l) + fi + set_xattr "$count" +} + +_init "$@" && main "$@" diff --git a/extras/ec-heal-script/gfid_needing_heal_parallel.sh b/extras/ec-heal-script/gfid_needing_heal_parallel.sh new file mode 100755 index 00000000000..d7f53c97c33 --- /dev/null +++ b/extras/ec-heal-script/gfid_needing_heal_parallel.sh @@ -0,0 +1,278 @@ +#!/bin/bash +# Copyright (c) 2019-2020 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + +# This script provides a list of all the files which can be healed or not healed. +# It also generates two files, potential_heal and can_not_heal, which contains the information +# of all theose files. These files could be used by correct_pending_heals.sh to correct +# the fragmnets so that files could be healed by shd. + +CAN_NOT_HEAL="can_not_heal" +CAN_HEAL="potential_heal" +LINE_SEP="===================================================" +LOG_DIR="/var/log/glusterfs" +LOG_FILE="$LOG_DIR/ec-heal-script.log" + +function heal_log() +{ + echo "$1" >> "$LOG_FILE" +} + +function _init () +{ + if [ $# -ne 1 ]; then + echo "usage: $0 <gluster volume name>"; + echo "This script provides a list of all the files which can be healed or not healed. +It also generates two files, potential_heal and can_not_heal, which contains the information +of all theose files. These files could be used by correct_pending_heals.sh to correct +the fragmnets so that files could be healed by shd." + exit 2; + fi + + volume=$1; +} + +function get_pending_entries () +{ + local volume_name=$1 + + gluster volume heal "$volume_name" info | grep -v ":/" | grep -v "Number of entries" | grep -v "Status:" | sort -u | sed '/^$/d' +} + +function get_entry_path_on_brick() +{ + local path="$1" + local gfid_string="" + if [[ "${path:0:1}" == "/" ]]; + then + echo "$path" + else + gfid_string="$(echo "$path" | cut -f2 -d':' | cut -f1 -d '>')" + echo "/.glusterfs/${gfid_string:0:2}/${gfid_string:2:2}/$gfid_string" + fi +} + +function run_command_on_server() +{ + local subvolume="$1" + local host="$2" + local cmd="$3" + local output + output=$(ssh -n "${host}" "${cmd}") + if [ -n "$output" ] + then + echo "$subvolume:$output" + fi +} + +function get_entry_path_all_bricks () +{ + local entry="$1" + local bricks="$2" + local cmd="" + for brick in $bricks + do + echo "${brick}#$(get_entry_path_on_brick "$entry")" + done | tr '\n' ',' +} + +function get_stat_for_entry_from_all_bricks () +{ + local entry="$1" + local bricks="$2" + local subvolume=0 + local host="" + local bpath="" + local cmd="" + + for brick in $bricks + do + if [[ "$((subvolume % 6))" == "0" ]] + then + subvolume=$((subvolume+1)) + fi + host=$(echo "$brick" | cut -f1 -d':') + bpath=$(echo "$brick" | cut -f2 -d':') + + cmd="stat --format=%F:%B:%s $bpath$(get_entry_path_on_brick "$entry") 2>/dev/null" + run_command_on_server "$subvolume" "${host}" "${cmd}" & + done | sort | uniq -c | sort -rnk1 +} + +function get_bricks_from_volume() +{ + local v=$1 + gluster volume info "$v" | grep -E "^Brick[0-9][0-9]*:" | cut -f2- -d':' +} + +function print_entry_gfid() +{ + local host="$1" + local dirpath="$2" + local entry="$3" + local gfid + gfid="$(ssh -n "${host}" "getfattr -d -m. -e hex $dirpath/$entry 2>/dev/null | grep trusted.gfid=|cut -f2 -d'='")" + echo "$entry" - "$gfid" +} + +function print_brick_directory_info() +{ + local h="$1" + local dirpath="$2" + while read -r e + do + print_entry_gfid "${h}" "${dirpath}" "${e}" + done < <(ssh -n "${h}" "ls $dirpath 2>/dev/null") +} + +function print_directory_info() +{ + local entry="$1" + local bricks="$2" + local h + local b + local gfid + for brick in $bricks; + do + h="$(echo "$brick" | cut -f1 -d':')" + b="$(echo "$brick" | cut -f2 -d':')" + dirpath="$b$(get_entry_path_on_brick "$entry")" + print_brick_directory_info "${h}" "${dirpath}" & + done | sort | uniq -c +} + +function print_entries_needing_heal() +{ + local quorum=0 + local entry="$1" + local bricks="$2" + while read -r line + do + quorum=$(echo "$line" | awk '{print $1}') + if [[ "$quorum" -lt 4 ]] + then + echo "$line - Not in Quorum" + else + echo "$line - In Quorum" + fi + done < <(print_directory_info "$entry" "$bricks") +} + +function find_file_paths () +{ + local bpath=$1 + local file_entry="" + local file_host="" + local cmd="" + local brick_root="" + local gfid="" + local actual_path="" + local gfid_path="" + + file_host=$(echo "$bpath" | cut -d ":" -f 1) + file_entry=$(echo "$bpath" | cut -d ":" -f 2) + brick_root=$(echo "$file_entry" | cut -d "#" -f 1) + + gfid=$(echo "${file_entry}" | grep ".glusterfs") + + if [[ -n "$gfid" ]] + then + gfid_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep -v '.glusterfs' " + actual_path=$(ssh -n "${file_host}" "${cmd}") + #removing absolute path so that user can refer this from mount point + actual_path=${actual_path#"$brick_root"} + else + actual_path=$(echo "$file_entry" | cut -d "#" -f 2) + file_entry=${file_entry//#} + cmd="find -L '$brick_root' -samefile '$file_entry' 2>/dev/null | grep '.glusterfs' " + gfid_path=$(ssh -n "${file_host}" "${cmd}") + gfid_path=${gfid_path#"$brick_root"} + fi + + echo "${gfid_path}|${actual_path}" +} + +function log_can_not_heal () +{ + local gfid_actual_paths=$1 + local file_paths=$2 + file_paths=${file_paths//#} + + echo "${LINE_SEP}" >> "$CAN_NOT_HEAL" + echo "Can Not Heal : $(echo "$gfid_actual_paths" | cut -d '|' -f 2)" >> "$CAN_NOT_HEAL" + for bpath in ${file_paths//,/ } + do + echo "${bpath}" >> "$CAN_NOT_HEAL" + done +} + +function main () +{ + local bricks="" + local quorum=0 + local stat_info="" + local file_type="" + local gfid_actual_paths="" + local bpath="" + local file_paths="" + local good=0 + local bad=0 + bricks=$(get_bricks_from_volume "$volume") + rm -f "$CAN_HEAL" + rm -f "$CAN_NOT_HEAL" + mkdir "$LOG_DIR" -p + + heal_log "Started $(basename "$BASH_SOURCE") on $(date) " + while read -r heal_entry + do + heal_log "------------------------------------------------------------------" + heal_log "$heal_entry" + + gfid_actual_paths="" + file_paths="$(get_entry_path_all_bricks "$heal_entry" "$bricks")" + stat_info="$(get_stat_for_entry_from_all_bricks "$heal_entry" "$bricks")" + heal_log "$stat_info" + + quorum=$(echo "$stat_info" | head -1 | awk '{print $1}') + good_stat=$(echo "$stat_info" | head -1 | awk '{print $3}') + file_type="$(echo "$stat_info" | head -1 | cut -f2 -d':')" + if [[ "$file_type" == "directory" ]] + then + print_entries_needing_heal "$heal_entry" "$bricks" + else + if [[ "$quorum" -ge 4 ]] + then + good=$((good + 1)) + heal_log "Verdict: Healable" + + echo "${good_stat}|$file_paths" >> "$CAN_HEAL" + else + bad=$((bad + 1)) + heal_log "Verdict: Not Healable" + for bpath in ${file_paths//,/ } + do + if [[ -z "$gfid_actual_paths" ]] + then + gfid_actual_paths=$(find_file_paths "$bpath") + else + break + fi + done + log_can_not_heal "$gfid_actual_paths" "${file_paths}" + fi + fi + done < <(get_pending_entries "$volume") + heal_log "=========================================" + heal_log "Total number of potential heal : ${good}" + heal_log "Total number of can not heal : ${bad}" + heal_log "=========================================" +} + +_init "$@" && main "$@" diff --git a/extras/failed-tests.py b/extras/failed-tests.py index 476e24331e5..f7f110246b5 100755 --- a/extras/failed-tests.py +++ b/extras/failed-tests.py @@ -1,5 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 +from __future__ import print_function import blessings import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning @@ -25,7 +26,7 @@ def process_failure(url, node): if t.find("Result: FAIL") != -1: for t2 in accum: if VERBOSE: - print t2.encode('utf-8') + print(t2.encode('utf-8')) if t2.find("Wstat") != -1: test_case = re.search('\./tests/.*\.t', t2) if test_case: @@ -69,26 +70,26 @@ def print_summary(failed_builds, total_builds, html=False): template = 0 if html: template = 1 - print render( + print(render( count[template], {'failed': failed_builds, 'total': total_builds} - ) - for k, v in summary.iteritems(): + )) + for k, v in summary.items(): if k == 'core': - print ''.join([TERM.red, "Found cores:", TERM.normal]) + print(''.join([TERM.red, "Found cores:", TERM.normal])) for comp, link in zip(v[::2], v[1::2]): - print render(component[template], {'comp': comp}) - print render( + print(render(component[template], {'comp': comp})) + print(render( regression_link[template], {'link': link[0], 'node': link[1]} - ) + )) else: - print render(failure_count[template], {'test': k, 'count': len(v)}) + print(render(failure_count[template], {'test': k, 'count': len(v)})) for link in v: - print render( + print(render( regression_link[template], {'link': link[0], 'node': link[1]} - ) + )) def get_summary(cut_off_date, reg_link): @@ -97,7 +98,7 @@ def get_summary(cut_off_date, reg_link): ''' success_count = 0 failure_count = 0 - for page in xrange(0, MAX_BUILDS, 100): + for page in range(0, MAX_BUILDS, 100): build_info = requests.get(''.join([ BASE, reg_link, @@ -114,11 +115,11 @@ def get_summary(cut_off_date, reg_link): success_count += 1 continue if VERBOSE: - print ''.join([ + print(''.join([ TERM.red, 'FAILURE on {0}'.format(build['url']), TERM.normal - ]) + ])) url = ''.join([build['url'], 'consoleText']) failure_count += 1 process_failure(url, build['builtOn']) diff --git a/extras/firewalld/Makefile.am b/extras/firewalld/Makefile.am index a5c11b0b783..530881fb8eb 100644 --- a/extras/firewalld/Makefile.am +++ b/extras/firewalld/Makefile.am @@ -1,6 +1,8 @@ EXTRA_DIST = glusterfs.xml if USE_FIREWALLD +if WITH_SERVER staticdir = /usr/lib/firewalld/services/ static_DATA = glusterfs.xml endif +endif diff --git a/extras/ganesha/Makefile.am b/extras/ganesha/Makefile.am new file mode 100644 index 00000000000..9eaa401b6c8 --- /dev/null +++ b/extras/ganesha/Makefile.am @@ -0,0 +1,2 @@ +SUBDIRS = scripts config ocf +CLEANFILES = diff --git a/extras/ganesha/config/Makefile.am b/extras/ganesha/config/Makefile.am new file mode 100644 index 00000000000..c729273096e --- /dev/null +++ b/extras/ganesha/config/Makefile.am @@ -0,0 +1,4 @@ +EXTRA_DIST= ganesha-ha.conf.sample + +confdir = $(sysconfdir)/ganesha +conf_DATA = ganesha-ha.conf.sample diff --git a/extras/ganesha/config/ganesha-ha.conf.sample b/extras/ganesha/config/ganesha-ha.conf.sample new file mode 100644 index 00000000000..c22892bde56 --- /dev/null +++ b/extras/ganesha/config/ganesha-ha.conf.sample @@ -0,0 +1,19 @@ +# Name of the HA cluster created. +# must be unique within the subnet +HA_NAME="ganesha-ha-360" +# +# N.B. you may use short names or long names; you may not use IP addrs. +# Once you select one, stay with it as it will be mildly unpleasant to +# clean up if you switch later on. Ensure that all names - short and/or +# long - are in DNS or /etc/hosts on all machines in the cluster. +# +# The subset of nodes of the Gluster Trusted Pool that form the ganesha +# HA cluster. Hostname is specified. +HA_CLUSTER_NODES="server1,server2,..." +#HA_CLUSTER_NODES="server1.lab.redhat.com,server2.lab.redhat.com,..." +# +# Virtual IPs for each of the nodes specified above. +VIP_server1="10.0.2.1" +VIP_server2="10.0.2.2" +#VIP_server1_lab_redhat_com="10.0.2.1" +#VIP_server2_lab_redhat_com="10.0.2.2" diff --git a/extras/ganesha/ocf/Makefile.am b/extras/ganesha/ocf/Makefile.am new file mode 100644 index 00000000000..990a609f254 --- /dev/null +++ b/extras/ganesha/ocf/Makefile.am @@ -0,0 +1,11 @@ +EXTRA_DIST= ganesha_grace ganesha_mon ganesha_nfsd + +# The root of the OCF resource agent hierarchy +# Per the OCF standard, it's always "lib", +# not "lib64" (even on 64-bit platforms). +ocfdir = $(prefix)/lib/ocf + +# The provider directory +radir = $(ocfdir)/resource.d/heartbeat + +ra_SCRIPTS = ganesha_grace ganesha_mon ganesha_nfsd diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace new file mode 100644 index 00000000000..825f7164597 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_grace @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "$OCF_DEBUG_LIBRARY" ]; then + . $OCF_DEBUG_LIBRARY +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_grace_active_default="grace-active" +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_grace"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="5s" /> +<action name="notify" timeout="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_grace_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_grace_start() +{ + local rc=${OCF_ERR_GENERIC} + local host=$(hostname -s) + + ocf_log debug "ganesha_grace_start()" + # give ganesha_mon RA a chance to set the crm_attr first + # I mislike the sleep, but it's not clear that looping + # with a small sleep is necessarily better + # start has a 40sec timeout, so a 5sec sleep here is okay + sleep 5 + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null ) + if [ $? -ne 0 ]; then + ocf_log info "grace start: crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # Three possibilities: + # 1. There is no attribute at all and attr_updater returns + # a zero length string. This happens when + # ganesha_mon::monitor hasn't run at least once to set + # the attribute. The assumption here is that the system + # is coming up. We pretend, for now, that the node is + # healthy, to allow the system to continue coming up. + # It will cure itself in a few seconds + # 2. There is an attribute, and it has the value "1"; this + # node is healthy. + # 3. There is an attribute, but it has no value or the value + # "0"; this node is not healthy. + + # case 1 + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + # case 2 + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + # case 3 + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_stop() +{ + + ocf_log debug "ganesha_grace_stop()" + return ${OCF_SUCCESS} +} + +ganesha_grace_notify() +{ + # since this is a clone RA we should only ever see pre-start + # or post-stop + mode="${OCF_RESKEY_CRM_meta_notify_type}-${OCF_RESKEY_CRM_meta_notify_operation}" + case "${mode}" in + pre-start | post-stop) + dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} + if [ $? -ne 0 ]; then + ocf_log info "dbus-send --print-reply --system --dest=org.ganesha.nfsd /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.grace string:${OCF_RESKEY_CRM_meta_notify_stop_uname} failed" + fi + ;; + esac + + return ${OCF_SUCCESS} +} + +ganesha_grace_monitor() +{ + local host=$(hostname -s) + + ocf_log debug "monitor" + + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + host=$(hostname) + attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null) + if [ $? -ne 0 ]; then + ocf_log info "crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed" + fi + fi + + # if there is no attribute (yet), maybe it's because + # this RA started before ganesha_mon (nfs-mon) has had + # chance to create it. In which case we'll pretend + # everything is okay this time around + if [[ -z "${attr}" ]]; then + return ${OCF_SUCCESS} + fi + + if [[ "${attr}" = *"value=1" ]]; then + return ${OCF_SUCCESS} + fi + + return ${OCF_NOT_RUNNING} +} + +ganesha_grace_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_grace_validate + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_grace_start + ;; +stop) ganesha_grace_stop + ;; +status|monitor) ganesha_grace_monitor + ;; +notify) ganesha_grace_notify + ;; +*) ganesha_grace_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon new file mode 100644 index 00000000000..2b4a9d6da84 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_mon @@ -0,0 +1,234 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +# Defaults +OCF_RESKEY_ganesha_active_default="ganesha-active" +OCF_RESKEY_grace_active_default="grace-active" +OCF_RESKEY_grace_delay_default="5" + +: ${OCF_RESKEY_ganesha_active=${OCF_RESKEY_ganesha_active_default}} +: ${OCF_RESKEY_grace_active=${OCF_RESKEY_grace_active_default}} +: ${OCF_RESKEY_grace_delay=${OCF_RESKEY_grace_delay_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_mon"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ganesha_active"> +<longdesc lang="en">NFS-Ganesha daemon active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha daemon active attribute</shortdesc> +<content type="string" default="ganesha-active" /> +</parameter> +<parameter name="grace_active"> +<longdesc lang="en">NFS-Ganesha grace active attribute</longdesc> +<shortdesc lang="en">NFS-Ganesha grace active attribute</shortdesc> +<content type="string" default="grace-active" /> +</parameter> +<parameter name="grace_delay"> +<longdesc lang="en"> +NFS-Ganesha grace delay. +When changing this, adjust the ganesha_grace RA's monitor interval to match. +</longdesc> +<shortdesc lang="en">NFS-Ganesha grace delay</shortdesc> +<content type="string" default="5" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="40s" /> +<action name="stop" timeout="40s" /> +<action name="status" timeout="20s" interval="60s" /> +<action name="monitor" depth="0" timeout="10s" interval="10s" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_mon_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case ${__OCF_ACTION} in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_mon_start() +{ + ocf_log debug "ganesha_mon_start" + ganesha_mon_monitor + return $OCF_SUCCESS +} + +ganesha_mon_stop() +{ + ocf_log debug "ganesha_mon_stop" + return $OCF_SUCCESS +} + +ganesha_mon_monitor() +{ + local host=$(hostname -s) + local pid_file="/var/run/ganesha.pid" + local rhel6_pid_file="/var/run/ganesha.nfsd.pid" + local proc_pid="/proc/" + + # RHEL6 /etc/init.d/nfs-ganesha adds -p /var/run/ganesha.nfsd.pid + # RHEL7 systemd does not. Would be nice if all distros used the + # same pid file. + if [ -e ${rhel6_pid_file} ]; then + pid_file=${rhel6_pid_file} + fi + if [ -e ${pid_file} ]; then + proc_pid="${proc_pid}$(cat ${pid_file})" + fi + + if [ "x${proc_pid}" != "x/proc/" -a -d ${proc_pid} ]; then + + attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_ganesha_active} -v 1 failed" + fi + + # ganesha_grace (nfs-grace) RA follows grace-active attr + # w/ constraint location + attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -n ${OCF_RESKEY_grace_active} -v 1 failed" + fi + + # ganesha_mon (nfs-mon) and ganesha_grace (nfs-grace) + # track grace-active crm_attr (attr != crm_attr) + # we can't just use the attr as there's no way to query + # its value in RHEL6 pacemaker + + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed" + fi + fi + + return ${OCF_SUCCESS} + fi + + # VIP fail-over is triggered by clearing the + # ganesha-active node attribute on this node. + # + # Meanwhile the ganesha_grace notify() runs when its + # nfs-grace resource is disabled on a node; which + # is triggered by clearing the grace-active attribute + # on this node. + # + # We need to allow time for it to run and put + # the remaining ganesha.nfsds into grace before + # initiating the VIP fail-over. + + attrd_updater -D -n ${OCF_RESKEY_grace_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_grace_active} failed" + fi + + host=$(hostname -s) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + host=$(hostname) + crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null + if [ $? -ne 0 ]; then + ocf_log info "mon monitor warning: crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 failed" + fi + fi + + sleep ${OCF_RESKEY_grace_delay} + + attrd_updater -D -n ${OCF_RESKEY_ganesha_active} + if [ $? -ne 0 ]; then + ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_ganesha_active} failed" + fi + + return ${OCF_SUCCESS} +} + +ganesha_mon_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_mon_validate + +# Translate each action into the appropriate function call +case ${__OCF_ACTION} in +start) ganesha_mon_start + ;; +stop) ganesha_mon_stop + ;; +status|monitor) ganesha_mon_monitor + ;; +*) ganesha_mon_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/ocf/ganesha_nfsd b/extras/ganesha/ocf/ganesha_nfsd new file mode 100644 index 00000000000..f91e8b6b8f7 --- /dev/null +++ b/extras/ganesha/ocf/ganesha_nfsd @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Copyright (c) 2014 Anand Subramanian anands@redhat.com +# Copyright (c) 2015 Red Hat Inc. +# All Rights Reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# + +# Initialization: +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +if [ -n "${OCF_DEBUG_LIBRARY}" ]; then + . ${OCF_DEBUG_LIBRARY} +else + : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} + . ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +fi + +OCF_RESKEY_ha_vol_mnt_default="/run/gluster/shared_storage" +: ${OCF_RESKEY_ha_vol_mnt=${OCF_RESKEY_ha_vol_mnt_default}} + +ganesha_meta_data() { + cat <<END +<?xml version="1.0"?> +<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd"> +<resource-agent name="ganesha_nfsd"> +<version>1.0</version> + +<longdesc lang="en"> +This Linux-specific resource agent acts as a dummy +resource agent for nfs-ganesha. +</longdesc> + +<shortdesc lang="en">Manages the user-space nfs-ganesha NFS server</shortdesc> + +<parameters> +<parameter name="ha_vol_mnt"> +<longdesc lang="en">HA State Volume Mount Point</longdesc> +<shortdesc lang="en">HA_State Volume Mount Point</shortdesc> +<content type="string" default="" /> +</parameter> +</parameters> + +<actions> +<action name="start" timeout="5s" /> +<action name="stop" timeout="5s" /> +<action name="status" depth="0" timeout="5s" interval="0" /> +<action name="monitor" depth="0" timeout="5s" interval="0" /> +<action name="meta-data" timeout="20s" /> +</actions> +</resource-agent> +END + +return ${OCF_SUCCESS} +} + +ganesha_nfsd_usage() { + echo "ganesha.nfsd USAGE" +} + +# Make sure meta-data and usage always succeed +case $__OCF_ACTION in + meta-data) ganesha_meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) ganesha_usage + exit ${OCF_SUCCESS} + ;; + *) + ;; +esac + +ganesha_nfsd_start() +{ + local long_host=$(hostname) + + if [[ -d /var/lib/nfs ]]; then + mv /var/lib/nfs /var/lib/nfs.backup + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs /var/lib/nfs.backup failed" + fi + ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "ln -s ${OCF_RESKEY_ha_vol_mnt}/nfs-ganesha/${long_host}/nfs /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_stop() +{ + + if [ -L /var/lib/nfs -a -d /var/lib/nfs.backup ]; then + rm -f /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "rm -f /var/lib/nfs failed" + fi + mv /var/lib/nfs.backup /var/lib/nfs + if [ $? -ne 0 ]; then + ocf_log notice "mv /var/lib/nfs.backup /var/lib/nfs failed" + fi + fi + + return ${OCF_SUCCESS} +} + +ganesha_nfsd_monitor() +{ + # pacemaker checks to see if RA is already running before starting it. + # if we return success, then it's presumed it's already running and + # doesn't need to be started, i.e. invoke the start action. + # return something other than success to make pacemaker invoke the + # start action + if [[ -L /var/lib/nfs ]]; then + return ${OCF_SUCCESS} + fi + return ${OCF_NOT_RUNNING} +} + +ganesha_nfsd_validate() +{ + return ${OCF_SUCCESS} +} + +ganesha_nfsd_validate + +# ocf_log notice "ganesha_nfsd ${OCF_RESOURCE_INSTANCE} $__OCF_ACTION" + +# Translate each action into the appropriate function call +case $__OCF_ACTION in +start) ganesha_nfsd_start + ;; +stop) ganesha_nfsd_stop + ;; +status|monitor) ganesha_nfsd_monitor + ;; +*) ganesha_nfsd_usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac + +rc=$? + +# The resource agent may optionally log a debug message +ocf_log debug "${OCF_RESOURCE_INSTANCE} ${__OCF_ACTION} returned $rc" +exit $rc diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am new file mode 100644 index 00000000000..7e345fd5f19 --- /dev/null +++ b/extras/ganesha/scripts/Makefile.am @@ -0,0 +1,6 @@ +EXTRA_DIST= create-export-ganesha.sh generate-epoch.py dbus-send.sh \ + ganesha-ha.sh + +scriptsdir = $(libexecdir)/ganesha +scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh generate-epoch.py \ + ganesha-ha.sh diff --git a/extras/ganesha/scripts/create-export-ganesha.sh b/extras/ganesha/scripts/create-export-ganesha.sh new file mode 100755 index 00000000000..3040e8138b0 --- /dev/null +++ b/extras/ganesha/scripts/create-export-ganesha.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +#This script is called by glusterd when the user +#tries to export a volume via NFS-Ganesha. +#An export file specific to a volume +#is created in GANESHA_DIR/exports. + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" +declare -i EXPORT_ID + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF + exit 1 + fi +} + + +if [ ! -d "$GANESHA_DIR/exports" ]; + then + mkdir $GANESHA_DIR/exports + check_cmd_status `echo $?` +fi + +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = "GLUSTER";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo ' Squash="No_root_squash";' +echo " Pseudo=\"/$VOL\";" +echo ' Protocols = "3", "4" ;' +echo ' Transports = "UDP","TCP";' +echo ' SecType = "sys";' +echo ' Security_Label = False;' +echo " }" +} +if [ "$OPTION" = "on" ]; +then + if ! (cat $CONF | grep $VOL.conf\"$ ) + then + write_conf $@ > $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF + count=`ls -l $GANESHA_DIR/exports/*.conf | wc -l` + if [ "$count" = "1" ] ; then + EXPORT_ID=2 + else + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + check_cmd_status `echo $?` + EXPORT_ID=EXPORT_ID+1 + sed -i s/Export_Id.*/"Export_Id= $EXPORT_ID ;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + check_cmd_status `echo $?` + fi + echo $EXPORT_ID > $GANESHA_DIR/.export_added + fi +else + rm -rf $GANESHA_DIR/exports/export.$VOL.conf + sed -i /$VOL.conf/d $CONF +fi diff --git a/extras/ganesha/scripts/dbus-send.sh b/extras/ganesha/scripts/dbus-send.sh new file mode 100755 index 00000000000..9d613a0e7ad --- /dev/null +++ b/extras/ganesha/scripts/dbus-send.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_DIR=${1%/} +OPTION=$2 +VOL=$3 +CONF=$GANESHA_DIR"/ganesha.conf" + +function check_cmd_status() +{ + if [ "$1" != "0" ] + then + logger "dynamic export failed on node :${hostname -s}" + fi +} + +#This function keeps track of export IDs and increments it with every new entry +function dynamic_export_add() +{ + dbus-send --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.AddExport string:$GANESHA_DIR/exports/export.$VOL.conf \ +string:"EXPORT(Path=/$VOL)" + check_cmd_status `echo $?` +} + +#This function removes an export dynamically(uses the export_id of the export) +function dynamic_export_remove() +{ + # Below bash fetch all the export from ShowExport command and search + # export entry based on path and then get its export entry. + # There are two possiblities for path, either entire volume will be + # exported or subdir. It handles both cases. But it remove only first + # entry from the list based on assumption that entry exported via cli + # has lowest export id value + removed_id=$(dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports | grep -B 1 -we \ + "/"$VOL -e "/"$VOL"/" | grep uint16 | awk '{print $2}' \ + | head -1) + + dbus-send --print-reply --system \ +--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ +org.ganesha.nfsd.exportmgr.RemoveExport uint16:$removed_id + check_cmd_status `echo $?` +} + +if [ "$OPTION" = "on" ]; +then + dynamic_export_add $@ +fi + +if [ "$OPTION" = "off" ]; +then + dynamic_export_remove $@ +fi diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh new file mode 100644 index 00000000000..9790a719e10 --- /dev/null +++ b/extras/ganesha/scripts/ganesha-ha.sh @@ -0,0 +1,1199 @@ +#!/bin/bash + +# Copyright 2015-2016 Red Hat Inc. All Rights Reserved +# +# Pacemaker+Corosync High Availability for NFS-Ganesha +# +# setup, teardown, add, delete, refresh-config, and status +# +# Each participating node in the cluster is assigned a virtual IP (VIP) +# which fails over to another node when its associated ganesha.nfsd dies +# for any reason. After the VIP is moved to another node all the +# ganesha.nfsds are send a signal using DBUS to put them into NFS GRACE. +# +# There are six resource agent types used: ganesha_mon, ganesha_grace, +# ganesha_nfsd, IPaddr, and Dummy. ganesha_mon is used to monitor the +# ganesha.nfsd. ganesha_grace is used to send the DBUS signal to put +# the remaining ganesha.nfsds into grace. ganesha_nfsd is used to start +# and stop the ganesha.nfsd during setup and teardown. IPaddr manages +# the VIP. A Dummy resource named $hostname-trigger_ip-1 is used to +# ensure that the NFS GRACE DBUS signal is sent after the VIP moves to +# the new host. + +GANESHA_HA_SH=$(realpath $0) +HA_NUM_SERVERS=0 +HA_SERVERS="" +HA_VOL_NAME="gluster_shared_storage" +HA_VOL_MNT="/run/gluster/shared_storage" +HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha" +SERVICE_MAN="DISTRO_NOT_FOUND" + +# rhel, fedora id, version +ID="" +VERSION_ID="" + +PCS9OR10_PCS_CNAME_OPTION="" +PCS9OR10_PCS_CLONE_OPTION="clone" +SECRET_PEM="/var/lib/glusterd/nfs/secret.pem" + +# UNBLOCK RA uses shared_storage which may become unavailable +# during any of the nodes reboot. Hence increase timeout value. +PORTBLOCK_UNBLOCK_TIMEOUT="60s" + +# Try loading the config from any of the distro +# specific configuration locations +if [ -f /etc/sysconfig/ganesha ] + then + . /etc/sysconfig/ganesha +fi +if [ -f /etc/conf.d/ganesha ] + then + . /etc/conf.d/ganesha +fi +if [ -f /etc/default/ganesha ] + then + . /etc/default/ganesha +fi + +GANESHA_CONF= + +function find_rhel7_conf +{ + while [[ $# > 0 ]] + do + key="$1" + case $key in + -f) + CONFFILE="$2" + break; + ;; + *) + ;; + esac + shift + done +} + +if [ -z ${CONFFILE} ] + then + find_rhel7_conf ${OPTIONS} + +fi + +GANESHA_CONF=${CONFFILE:-/etc/ganesha/ganesha.conf} + +usage() { + + echo "Usage : add|delete|refresh-config|status" + echo "Add-node : ganesha-ha.sh --add <HA_CONF_DIR> \ +<NODE-HOSTNAME> <NODE-VIP>" + echo "Delete-node: ganesha-ha.sh --delete <HA_CONF_DIR> \ +<NODE-HOSTNAME>" + echo "Refresh-config : ganesha-ha.sh --refresh-config <HA_CONFDIR> \ +<volume>" + echo "Status : ganesha-ha.sh --status <HA_CONFDIR>" +} + +determine_service_manager () { + + if [ -e "/bin/systemctl" ]; + then + SERVICE_MAN="/bin/systemctl" + elif [ -e "/sbin/invoke-rc.d" ]; + then + SERVICE_MAN="/sbin/invoke-rc.d" + elif [ -e "/sbin/service" ]; + then + SERVICE_MAN="/sbin/service" + fi + if [[ "${SERVICE_MAN}X" == "DISTRO_NOT_FOUNDX" ]] + then + logger "Service manager not recognized, exiting" + exit 1 + fi +} + +manage_service () +{ + local action=${1} + local new_node=${2} + local option= + + if [[ "${action}" == "start" ]]; then + option="yes" + else + option="no" + fi + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${GANESHA_HA_SH} --setup-ganesha-conf-files $HA_CONFDIR $option" + + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]] + then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} ${action} nfs-ganesha" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${new_node} "${SERVICE_MAN} nfs-ganesha ${action}" + fi +} + + +check_cluster_exists() +{ + local name=${1} + local cluster_name="" + + if [ -e /var/run/corosync.pid ]; then + cluster_name=$(pcs status | grep "Cluster name:" | cut -d ' ' -f 3) + if [[ "${cluster_name}X" == "${name}X" ]]; then + logger "$name already exists, exiting" + exit 0 + fi + fi +} + + +determine_servers() +{ + local cmd=${1} + local num_servers=0 + local tmp_ifs=${IFS} + local ha_servers="" + + if [ "${cmd}X" != "setupX" -a "${cmd}X" != "statusX" ]; then + ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//') + IFS=$' ' + for server in ${ha_servers} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${ha_servers}" + else + IFS=$',' + for server in ${HA_CLUSTER_NODES} ; do + num_servers=$(expr ${num_servers} + 1) + done + IFS=${tmp_ifs} + HA_NUM_SERVERS=${num_servers} + HA_SERVERS="${HA_CLUSTER_NODES//,/ }" + fi +} + +stop_ganesha_all() +{ + local serverlist=${1} + for node in ${serverlist} ; do + manage_service "stop" ${node} + done +} + +setup_cluster() +{ + local name=${1} + local num_servers=${2} + local servers=${3} + local unclean="" + local quorum_policy="stop" + + logger "setting up cluster ${name} with the following ${servers}" + + # pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} ${servers} + pcs cluster setup --force ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} + if [ $? -ne 0 ]; then + logger "pcs cluster setup ${PCS9OR10_PCS_CNAME_OPTION} ${name} --enable ${servers} failed, shutting down ganesha and bailing out" + #set up failed stop all ganesha process and clean up symlinks in cluster + stop_ganesha_all "${servers}" + exit 1; + fi + + # pcs cluster auth ${servers} + pcs cluster auth + if [ $? -ne 0 ]; then + logger "pcs cluster auth failed" + fi + + pcs cluster start --all + if [ $? -ne 0 ]; then + logger "pcs cluster start failed" + exit 1; + fi + + sleep 1 + # wait for the cluster to elect a DC before querying or writing + # to the CIB. BZ 1334092 + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + while [ $? -ne 0 ]; do + crmadmin --dc_lookup --timeout=5000 > /dev/null 2>&1 + done + + unclean=$(pcs status | grep -u "UNCLEAN") + while [[ "${unclean}X" == "UNCLEANX" ]]; do + sleep 1 + unclean=$(pcs status | grep -u "UNCLEAN") + done + sleep 1 + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi + + pcs property set stonith-enabled=false + if [ $? -ne 0 ]; then + logger "warning: pcs property set stonith-enabled=false failed" + fi +} + + +setup_finalize_ha() +{ + local cibfile=${1} + local stopped="" + + stopped=$(pcs status | grep -u "Stopped") + while [[ "${stopped}X" == "StoppedX" ]]; do + sleep 1 + stopped=$(pcs status | grep -u "Stopped") + done +} + + +refresh_config () +{ + local short_host=$(hostname -s) + local VOL=${1} + local HA_CONFDIR=${2} + local short_host=$(hostname -s) + + local export_id=$(grep ^[[:space:]]*Export_Id $HA_CONFDIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + + + if [ -e ${SECRET_PEM} ]; then + while [[ ${3} ]]; do + current_host=`echo ${3} | cut -d "." -f 1` + if [[ ${short_host} != ${current_host} ]]; then + output=$(ssh -oPasswordAuthentication=no \ +-oStrictHostKeyChecking=no -i ${SECRET_PEM} root@${current_host} \ +"dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:\"EXPORT(Export_Id=$export_id)\" 2>&1") + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ]; then + echo "Refresh-config failed on ${current_host}. Please check logs on ${current_host}" + else + echo "Refresh-config completed on ${current_host}." + fi + + fi + shift + done + else + echo "Error: refresh-config failed. Passwordless ssh is not enabled." + exit 1 + fi + + # Run the same command on the localhost, + output=$(dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.UpdateExport \ +string:$HA_CONFDIR/exports/export.$VOL.conf \ +string:"EXPORT(Export_Id=$export_id)" 2>&1) + ret=$? + logger <<< "${output}" + if [ ${ret} -ne 0 ] ; then + echo "Refresh-config failed on localhost." + else + echo "Success: refresh-config completed." + fi +} + + +teardown_cluster() +{ + local name=${1} + + for server in ${HA_SERVERS} ; do + if [[ ${HA_CLUSTER_NODES} != *${server}* ]]; then + logger "info: ${server} is not in config, removing" + + pcs cluster stop ${server} --force + if [ $? -ne 0 ]; then + logger "warning: pcs cluster stop ${server} failed" + fi + + pcs cluster node remove ${server} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${server} failed" + fi + fi + done + + # BZ 1193433 - pcs doesn't reload cluster.conf after modification + # after teardown completes, a subsequent setup will appear to have + # 'remembered' the deleted node. You can work around this by + # issuing another `pcs cluster node remove $node`, + # `crm_node -f -R $server`, or + # `cibadmin --delete --xml-text '<node id="$server" + # uname="$server"/>' + + pcs cluster stop --all + if [ $? -ne 0 ]; then + logger "warning pcs cluster stop --all failed" + fi + + pcs cluster destroy + if [ $? -ne 0 ]; then + logger "error pcs cluster destroy failed" + exit 1 + fi +} + + +cleanup_ganesha_config () +{ + rm -f /etc/corosync/corosync.conf + rm -rf /etc/cluster/cluster.conf* + rm -rf /var/lib/pacemaker/cib/* + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf +} + +do_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local weight="1000" + + # first a constraint location rule that says the VIP must be where + # there's a ganesha.nfsd running + pcs -f ${cibfile} constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group rule score=-INFINITY ganesha-active ne 1 failed" + fi + + # then a set of constraint location prefers to set the prefered order + # for where a VIP should move + while [[ ${1} ]]; do + pcs -f ${cibfile} constraint location ${primary}-group prefers ${1}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${1}=${weight} failed" + fi + weight=$(expr ${weight} + 1000) + shift + done + # and finally set the highest preference for the VIP to its home node + # default weight when created is/was 100. + # on Fedora setting appears to be additive, so to get the desired + # value we adjust the weight + # weight=$(expr ${weight} - 100) + pcs -f ${cibfile} constraint location ${primary}-group prefers ${primary}=${weight} + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location ${primary}-group prefers ${primary}=${weight} failed" + fi +} + + +wrap_create_virt_ip_constraints() +{ + local cibfile=${1}; shift + local primary=${1}; shift + local head="" + local tail="" + + # build a list of peers, e.g. for a four node cluster, for node1, + # the result is "node2 node3 node4"; for node2, "node3 node4 node1" + # and so on. + while [[ ${1} ]]; do + if [[ ${1} == ${primary} ]]; then + shift + while [[ ${1} ]]; do + tail=${tail}" "${1} + shift + done + else + head=${head}" "${1} + fi + shift + done + do_create_virt_ip_constraints ${cibfile} ${primary} ${tail} ${head} +} + + +create_virt_ip_constraints() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + wrap_create_virt_ip_constraints ${cibfile} ${1} ${HA_SERVERS} + shift + done +} + + +setup_create_resources() +{ + local cibfile=$(mktemp -u) + + # fixup /var/lib/nfs + logger "pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION}" + pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs_setup ocf:heartbeat:ganesha_nfsd ha_vol_mnt=${HA_VOL_MNT} ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace + # start method. Allow time for ganesha_mon to start and set the + # ganesha-active crm_attribute + sleep 5 + + pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} notify=true + if [ $? -ne 0 ]; then + logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace ${PCS9OR10_PCS_CLONE_OPTION} failed" + fi + + pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint location nfs-grace-clone rule score=-INFINITY grace-active ne 1" + fi + + pcs cluster cib ${cibfile} + + while [[ ${1} ]]; do + + # this is variable indirection + # from a nvs like 'VIP_host1=10.7.6.5' or 'VIP_host1="10.7.6.5"' + # (or VIP_host-1=..., or VIP_host-1.my.domain.name=...) + # a variable 'clean_name' is created (e.g. w/ value 'VIP_host_1') + # and a clean nvs (e.g. w/ value 'VIP_host_1="10_7_6_5"') + # after the `eval ${clean_nvs}` there is a variable VIP_host_1 + # with the value '10_7_6_5', and the following \$$ magic to + # reference it, i.e. `eval tmp_ipaddr=\$${clean_name}` gives us + # ${tmp_ipaddr} with 10_7_6_5 and then convert the _s back to .s + # to give us ipaddr="10.7.6.5". whew! + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + + shift + done + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +teardown_resources() +{ + # local mntpt=$(grep ha-vol-mnt ${HA_CONFIG_FILE} | cut -d = -f 2) + + # restore /var/lib/nfs + logger "notice: pcs resource delete nfs_setup-clone" + pcs resource delete nfs_setup-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs_setup-clone failed" + fi + + # delete -clone resource agents + # in particular delete the ganesha monitor so we don't try to + # trigger anything when we shut down ganesha next. + pcs resource delete nfs-mon-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-mon-clone failed" + fi + + pcs resource delete nfs-grace-clone + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete nfs-grace-clone failed" + fi + + while [[ ${1} ]]; do + pcs resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs resource delete ${1}-group failed" + fi + shift + done + +} + + +recreate_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + # this is variable indirection + # see the comment on the same a few lines up + name="VIP_${1}" + clean_name=${name//[-.]/_} + nvs=$(grep "^${name}=" ${HA_CONFDIR}/ganesha-ha.conf) + clean_nvs=${nvs//[-.]/_} + eval ${clean_nvs} + eval tmp_ipaddr=\$${clean_name} + ipaddr=${tmp_ipaddr//_/.} + + pcs -f ${cibfile} resource create ${1}-nfs_block ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=block ip=${ipaddr} --group ${1}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s --group ${1}-group --after ${1}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-cluster_ip-1 ocf:heartbeat:IPaddr ip=${ipaddr} \ + cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${1}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${1}-cluster_ip-1 failed" + fi + + pcs -f ${cibfile} resource create ${1}-nfs_unblock ocf:heartbeat:portblock protocol=tcp \ + portno=2049 action=unblock ip=${ipaddr} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${1}-group --after ${1}-cluster_ip-1 \ + op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} \ + op monitor interval=10s timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${1}-nfs_unblock failed" + fi + + shift + done +} + + +addnode_recreate_resources() +{ + local cibfile=${1}; shift + local add_node=${1}; shift + local add_vip=${1}; shift + + recreate_resources ${cibfile} ${HA_SERVERS} + + pcs -f ${cibfile} resource create ${add_node}-nfs_block ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=block ip=${add_vip} --group ${add_node}-group + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_block failed" + fi + pcs -f ${cibfile} resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s --group ${add_node}-group \ + --after ${add_node}-nfs_block + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-cluster_ip-1 ocf:heartbeat:IPaddr \ + ip=${add_vip} cidr_netmask=32 op monitor interval=15s failed" + fi + + pcs -f ${cibfile} constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 + if [ $? -ne 0 ]; then + logger "warning: pcs constraint order nfs-grace-clone then ${add_node}-cluster_ip-1 failed" + fi + pcs -f ${cibfile} resource create ${add_node}-nfs_unblock ocf:heartbeat:portblock \ + protocol=tcp portno=2049 action=unblock ip=${add_vip} reset_local_on_unblock_stop=true \ + tickle_dir=${HA_VOL_MNT}/nfs-ganesha/tickle_dir/ --group ${add_node}-group --after \ + ${add_node}-cluster_ip-1 op stop timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op start \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} op monitor interval=10s \ + timeout=${PORTBLOCK_UNBLOCK_TIMEOUT} + if [ $? -ne 0 ]; then + logger "warning pcs resource create ${add_node}-nfs_unblock failed" + fi +} + + +clear_resources() +{ + local cibfile=${1}; shift + + while [[ ${1} ]]; do + pcs -f ${cibfile} resource delete ${1}-group + if [ $? -ne 0 ]; then + logger "warning: pcs -f ${cibfile} resource delete ${1}-group" + fi + + shift + done +} + + +addnode_create_resources() +{ + local add_node=${1}; shift + local add_vip=${1}; shift + local cibfile=$(mktemp -u) + + # start HA on the new node + pcs cluster start ${add_node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster start ${add_node} failed" + fi + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 resources, clearing + # their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + addnode_recreate_resources ${cibfile} ${add_node} ${add_vip} + + HA_SERVERS="${HA_SERVERS} ${add_node}" + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} +} + + +deletenode_delete_resources() +{ + local node=${1}; shift + local ha_servers=$(echo "${HA_SERVERS}" | sed s/${node}//) + local cibfile=$(mktemp -u) + + pcs cluster cib ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib ${cibfile} failed" + fi + + # delete all the -cluster_ip-1 and -trigger_ip-1 resources, + # clearing their constraints, then create them again so we can + # recompute their constraints + clear_resources ${cibfile} ${HA_SERVERS} + recreate_resources ${cibfile} ${ha_servers} + HA_SERVERS=$(echo "${ha_servers}" | sed -e "s/ / /") + + create_virt_ip_constraints ${cibfile} ${HA_SERVERS} + + pcs cluster cib-push ${cibfile} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster cib-push ${cibfile} failed" + fi + rm -f ${cibfile} + +} + + +deletenode_update_haconfig() +{ + local name="VIP_${1}" + local clean_name=${name//[-.]/_} + + ha_servers=$(echo ${HA_SERVERS} | sed -e "s/ /,/") + sed -i -e "s/^HA_CLUSTER_NODES=.*$/HA_CLUSTER_NODES=\"${ha_servers// /,}\"/" -e "s/^${name}=.*$//" -e "/^$/d" ${HA_CONFDIR}/ganesha-ha.conf +} + + +setup_state_volume() +{ + local mnt=${HA_VOL_MNT} + local longname="" + local shortname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + while [[ ${1} ]]; do + + if [[ ${1} == *${dname} ]]; then + dirname=${1} + else + dirname=${1}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/tickle_dir ]; then + mkdir ${mnt}/nfs-ganesha/tickle_dir + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + fi + done + shift + done + +} + + +enable_pacemaker() +{ + while [[ ${1} ]]; do + if [[ "${SERVICE_MAN}" == "/bin/systemctl" ]]; then + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} enable pacemaker" + else + ssh -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \ +${SECRET_PEM} root@${1} "${SERVICE_MAN} pacemaker enable" + fi + shift + done +} + + +addnode_state_volume() +{ + local newnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${newnode} == *${dname} ]]; then + dirname=${newnode} + else + dirname=${newnode}${dname} + fi + + if [ ! -d ${mnt}/nfs-ganesha/${dirname} ]; then + mkdir ${mnt}/nfs-ganesha/${dirname} + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/state + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/state + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4recov + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/v4old + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm + fi + if [ ! -d ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak ]; then + mkdir ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + chown rpcuser:rpcuser ${mnt}/nfs-ganesha/${dirname}/nfs/statd/sm.bak + fi + if [ ! -e ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state ]; then + touch ${mnt}/nfs-ganesha/${dirname}/nfs/statd/state + fi + + for server in ${HA_SERVERS} ; do + + if [[ ${server} != ${dirname} ]]; then + ln -s ${mnt}/nfs-ganesha/${server}/nfs/ganesha ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha/${server} + ln -s ${mnt}/nfs-ganesha/${server}/nfs/statd ${mnt}/nfs-ganesha/${dirname}/nfs/statd/${server} + + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/ganesha ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + ln -s ${mnt}/nfs-ganesha/${dirname}/nfs/statd ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done + +} + + +delnode_state_volume() +{ + local delnode=${1}; shift + local mnt=${HA_VOL_MNT} + local longname="" + local dname="" + local dirname="" + + longname=$(hostname) + dname=${longname#$(hostname -s)} + + if [[ ${delnode} == *${dname} ]]; then + dirname=${delnode} + else + dirname=${delnode}${dname} + fi + + rm -rf ${mnt}/nfs-ganesha/${dirname} + + for server in ${HA_SERVERS} ; do + if [[ ${server} != ${dirname} ]]; then + rm -f ${mnt}/nfs-ganesha/${server}/nfs/ganesha/${dirname} + rm -f ${mnt}/nfs-ganesha/${server}/nfs/statd/${dirname} + fi + done +} + + +status() +{ + local scratch=$(mktemp) + local regex_str="^${1}-cluster_ip-1" + local healthy=0 + local index=1 + local nodes + + # change tabs to spaces, strip leading spaces, including any + # new '*' at the beginning of a line introduced in pcs-0.10.x + pcs status | sed -e "s/\t/ /g" -e "s/^[ ]*\*//" -e "s/^[ ]*//" > ${scratch} + + nodes[0]=${1}; shift + + # make a regex of the configured nodes + # and initalize the nodes array for later + while [[ ${1} ]]; do + + regex_str="${regex_str}|^${1}-cluster_ip-1" + nodes[${index}]=${1} + ((index++)) + shift + done + + # print the nodes that are expected to be online + grep -E "Online:" ${scratch} + + echo + + # print the VIPs and which node they are on + grep -E "${regex_str}" < ${scratch} | cut -d ' ' -f 1,4 + + echo + + # check if the VIP and port block/unblock RAs are on the expected nodes + for n in ${nodes[*]}; do + + grep -E -x "${n}-nfs_block \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-cluster_ip-1 \(ocf::heartbeat:IPaddr\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + grep -E -x "${n}-nfs_unblock \(ocf::heartbeat:portblock\): Started ${n}" > /dev/null 2>&1 ${scratch} + result=$? + ((healthy+=${result})) + done + + grep -E "\):\ Stopped|FAILED" > /dev/null 2>&1 ${scratch} + result=$? + + if [ ${result} -eq 0 ]; then + echo "Cluster HA Status: BAD" + elif [ ${healthy} -eq 0 ]; then + echo "Cluster HA Status: HEALTHY" + else + echo "Cluster HA Status: FAILOVER" + fi + + rm -f ${scratch} +} + +create_ganesha_conf_file() +{ + if [[ "$1" == "yes" ]]; + then + if [ -e $GANESHA_CONF ]; + then + rm -rf $GANESHA_CONF + fi + # The symlink /etc/ganesha/ganesha.conf need to be + # created using ganesha conf file mentioned in the + # shared storage. Every node will only have this + # link and actual file will stored in shared storage, + # so that ganesha conf editing of ganesha conf will + # be easy as well as it become more consistent. + + ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF + else + # Restoring previous file + rm -rf $GANESHA_CONF + cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF + sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $GANESHA_CONF + fi +} + +set_quorum_policy() +{ + local quorum_policy="stop" + local num_servers=${1} + + if [ ${num_servers} -lt 3 ]; then + quorum_policy="ignore" + fi + pcs property set no-quorum-policy=${quorum_policy} + if [ $? -ne 0 ]; then + logger "warning: pcs property set no-quorum-policy=${quorum_policy} failed" + fi +} + +main() +{ + + local cmd=${1}; shift + if [[ ${cmd} == *help ]]; then + usage + exit 0 + fi + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --on + fi + + local osid="" + + osid=$(grep ^ID= /etc/os-release) + eval $(echo ${osid} | grep -F ID=) + osid=$(grep ^VERSION_ID= /etc/os-release) + eval $(echo ${osid} | grep -F VERSION_ID=) + + HA_CONFDIR=${1%/}; shift + local ha_conf=${HA_CONFDIR}/ganesha-ha.conf + local node="" + local vip="" + + # ignore any comment lines + cfgline=$(grep ^HA_NAME= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_NAME=) + cfgline=$(grep ^HA_CLUSTER_NODES= ${ha_conf}) + eval $(echo ${cfgline} | grep -F HA_CLUSTER_NODES=) + + case "${cmd}" in + + setup | --setup) + logger "setting up ${HA_NAME}" + + check_cluster_exists ${HA_NAME} + + determine_servers "setup" + + # Fedora 29+ and rhel/centos 8 has PCS-0.10.x + # default is pcs-0.10.x options but check for + # rhel/centos 7 (pcs-0.9.x) and adjust accordingly + if [[ ! ${ID} =~ {rhel,centos} ]]; then + if [[ ${VERSION_ID} == 7.* ]]; then + PCS9OR10_PCS_CNAME_OPTION="--name" + PCS9OR10_PCS_CLONE_OPTION="--clone" + fi + fi + + if [[ "${HA_NUM_SERVERS}X" != "1X" ]]; then + + determine_service_manager + + setup_cluster ${HA_NAME} ${HA_NUM_SERVERS} "${HA_SERVERS}" + + setup_create_resources ${HA_SERVERS} + + setup_finalize_ha + + setup_state_volume ${HA_SERVERS} + + enable_pacemaker ${HA_SERVERS} + + else + + logger "insufficient servers for HA, aborting" + fi + ;; + + teardown | --teardown) + logger "tearing down ${HA_NAME}" + + determine_servers "teardown" + + teardown_resources ${HA_SERVERS} + + teardown_cluster ${HA_NAME} + + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + cleanup | --cleanup) + cleanup_ganesha_config ${HA_CONFDIR} + ;; + + add | --add) + node=${1}; shift + vip=${1}; shift + + logger "adding ${node} with ${vip} to ${HA_NAME}" + + determine_service_manager + + manage_service "start" ${node} + + determine_servers "add" + + pcs cluster node add ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node add ${node} failed" + fi + + addnode_create_resources ${node} ${vip} + # Subsequent add-node recreates resources for all the nodes + # that already exist in the cluster. The nodes are picked up + # from the entries in the ganesha-ha.conf file. Adding the + # newly added node to the file so that the resources specfic + # to this node is correctly recreated in the future. + clean_node=${node//[-.]/_} + echo "VIP_${node}=\"${vip}\"" >> ${HA_CONFDIR}/ganesha-ha.conf + + NEW_NODES="$HA_CLUSTER_NODES,${node}" + + sed -i s/HA_CLUSTER_NODES.*/"HA_CLUSTER_NODES=\"$NEW_NODES\""/ \ +$HA_CONFDIR/ganesha-ha.conf + + addnode_state_volume ${node} + + # addnode_create_resources() already appended ${node} to + # HA_SERVERS, so only need to increment HA_NUM_SERVERS + # and set quorum policy + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} + 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + delete | --delete) + node=${1}; shift + + logger "deleting ${node} from ${HA_NAME}" + + determine_servers "delete" + + deletenode_delete_resources ${node} + + pcs cluster node remove ${node} + if [ $? -ne 0 ]; then + logger "warning: pcs cluster node remove ${node} failed" + fi + + deletenode_update_haconfig ${node} + + delnode_state_volume ${node} + + determine_service_manager + + manage_service "stop" ${node} + + HA_NUM_SERVERS=$(expr ${HA_NUM_SERVERS} - 1) + set_quorum_policy ${HA_NUM_SERVERS} + ;; + + status | --status) + determine_servers "status" + + status ${HA_SERVERS} + ;; + + refresh-config | --refresh-config) + VOL=${1} + + determine_servers "refresh-config" + + refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS} + ;; + + setup-ganesha-conf-files | --setup-ganesha-conf-files) + + create_ganesha_conf_file ${1} + ;; + + *) + # setup and teardown are not intended to be used by a + # casual user + usage + logger "Usage: ganesha-ha.sh add|delete|status" + ;; + + esac + + if (selinuxenabled) ;then + semanage boolean -m gluster_use_execmem --off + fi +} + +main $* diff --git a/extras/ganesha/scripts/generate-epoch.py b/extras/ganesha/scripts/generate-epoch.py new file mode 100755 index 00000000000..77af014bab9 --- /dev/null +++ b/extras/ganesha/scripts/generate-epoch.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. +# +# Generates unique epoch value on each gluster node to be used by +# nfs-ganesha service on that node. +# +# Configure 'EPOCH_EXEC' option to this script path in +# '/etc/sysconfig/ganesha' file used by nfs-ganesha service. +# +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid + +import time +import binascii + +# Calculate the now() time into a 64-bit integer value +def epoch_now(): + epoch_time = int(time.mktime(time.localtime())) << 32 + return epoch_time + +# Read glusterd UUID and extract first 32-bit of it +def epoch_uuid(): + file_name = '/var/lib/glusterd/glusterd.info' + + for line in open(file_name): + if "UUID" in line: + glusterd_uuid = line.split('=')[1].strip() + + uuid_bin = binascii.unhexlify(glusterd_uuid.replace("-","")) + + epoch_uuid = int(binascii.hexlify(uuid_bin), 32) & 0xFFFF0000 + return epoch_uuid + +# Construct epoch as follows - +# first 32-bit contains the now() time +# rest 32-bit value contains the local glusterd node uuid +epoch = (epoch_now() | epoch_uuid()) +print((str(epoch))) + +exit(0) diff --git a/extras/geo-rep/Makefile.am b/extras/geo-rep/Makefile.am index e4603ae80b8..09eff308ac4 100644 --- a/extras/geo-rep/Makefile.am +++ b/extras/geo-rep/Makefile.am @@ -1,4 +1,4 @@ -scriptsdir = $(datadir)/glusterfs/scripts +scriptsdir = $(libexecdir)/glusterfs/scripts scripts_SCRIPTS = gsync-upgrade.sh generate-gfid-file.sh get-gfid.sh \ slave-upgrade.sh schedule_georep.py diff --git a/extras/geo-rep/gsync-sync-gfid.c b/extras/geo-rep/gsync-sync-gfid.c index e9b9e633402..47dca0413e9 100644 --- a/extras/geo-rep/gsync-sync-gfid.c +++ b/extras/geo-rep/gsync-sync-gfid.c @@ -7,103 +7,103 @@ #include <libgen.h> #include <ctype.h> #include <stdlib.h> -#include "glusterfs.h" -#include "syscall.h" +#include <glusterfs/glusterfs.h> +#include <glusterfs/syscall.h> #ifndef UUID_CANONICAL_FORM_LEN #define UUID_CANONICAL_FORM_LEN 36 #endif #ifndef GF_FUSE_AUX_GFID_HEAL -#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal" +#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal" #endif -#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN)) +#define GLFS_LINE_MAX (PATH_MAX + (2 * UUID_CANONICAL_FORM_LEN)) int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - char *file = NULL; - char *tmp = NULL; - char *tmp1 = NULL; - char *parent_dir = NULL; - char *gfid = NULL; - char *bname = NULL; - int ret = -1; - int len = 0; - FILE *fp = NULL; - char line[GLFS_LINE_MAX] = {0,}; - char *path = NULL; - void *blob = NULL; - void *tmp_blob = NULL; - - if (argc != 2) { - /* each line in the file has the following format - * uuid-in-canonical-form path-relative-to-gluster-mount. - * Both uuid and relative path are from master mount. - */ - fprintf (stderr, "usage: %s <file-of-paths-to-be-synced>\n", - argv[0]); - goto out; + char *file = NULL; + char *tmp = NULL; + char *tmp1 = NULL; + char *parent_dir = NULL; + char *gfid = NULL; + char *bname = NULL; + int ret = -1; + int len = 0; + FILE *fp = NULL; + char line[GLFS_LINE_MAX] = { + 0, + }; + char *path = NULL; + void *blob = NULL; + void *tmp_blob = NULL; + + if (argc != 2) { + /* each line in the file has the following format + * uuid-in-canonical-form path-relative-to-gluster-mount. + * Both uuid and relative path are from master mount. + */ + fprintf(stderr, "usage: %s <file-of-paths-to-be-synced>\n", argv[0]); + goto out; + } + + file = argv[1]; + + fp = fopen(file, "r"); + if (fp == NULL) { + fprintf(stderr, "cannot open %s for reading (%s)\n", file, + strerror(errno)); + goto out; + } + + while (fgets(line, GLFS_LINE_MAX, fp) != NULL) { + tmp = line; + path = gfid = line; + + path += UUID_CANONICAL_FORM_LEN + 1; + + while (isspace(*path)) + path++; + + len = strlen(line); + if ((len < GLFS_LINE_MAX) && (line[len - 1] == '\n')) + line[len - 1] = '\0'; + + line[UUID_CANONICAL_FORM_LEN] = '\0'; + + tmp = strdup(path); + tmp1 = strdup(path); + parent_dir = dirname(tmp); + bname = basename(tmp1); + + /* gfid + '\0' + bname + '\0' */ + len = UUID_CANONICAL_FORM_LEN + 1 + strlen(bname) + 1; + + blob = malloc(len); + + memcpy(blob, gfid, UUID_CANONICAL_FORM_LEN); + + tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1; + + memcpy(tmp_blob, bname, strlen(bname)); + + ret = sys_lsetxattr(parent_dir, GF_FUSE_AUX_GFID_HEAL, blob, len, 0); + if (ret < 0) { + fprintf(stderr, "setxattr on %s/%s failed (%s)\n", parent_dir, + bname, strerror(errno)); } + memset(line, 0, GLFS_LINE_MAX); - file = argv[1]; + free(blob); + free(tmp); + free(tmp1); + blob = NULL; + } - fp = fopen (file, "r"); - if (fp == NULL) { - fprintf (stderr, "cannot open %s for reading (%s)\n", - file, strerror (errno)); - goto out; - } - - while (fgets (line, GLFS_LINE_MAX, fp) != NULL) { - tmp = line; - path = gfid = line; - - path += UUID_CANONICAL_FORM_LEN + 1; - - while(isspace (*path)) - path++; - - if ((strlen (line) < GLFS_LINE_MAX) && - (line[strlen (line) - 1] == '\n')) - line[strlen (line) - 1] = '\0'; - - line[UUID_CANONICAL_FORM_LEN] = '\0'; - - tmp = strdup (path); - tmp1 = strdup (path); - parent_dir = dirname (tmp); - bname = basename (tmp1); - - /* gfid + '\0' + bname + '\0' */ - len = UUID_CANONICAL_FORM_LEN + 1 + strlen (bname) + 1; - - blob = calloc (1, len); - - memcpy (blob, gfid, UUID_CANONICAL_FORM_LEN); - - tmp_blob = blob + UUID_CANONICAL_FORM_LEN + 1; - - memcpy (tmp_blob, bname, strlen (bname)); - - ret = sys_lsetxattr (parent_dir, GF_FUSE_AUX_GFID_HEAL, - blob, len, 0); - if (ret < 0) { - fprintf (stderr, "setxattr on %s/%s failed (%s)\n", - parent_dir, bname, strerror (errno)); - } - memset (line, 0, GLFS_LINE_MAX); - - free (blob); - free (tmp); free (tmp1); - blob = NULL; - } - - ret = 0; + ret = 0; out: - if (fp) - fclose(fp); - return ret; + if (fp) + fclose(fp); + return ret; } - diff --git a/extras/geo-rep/schedule_georep.py.in b/extras/geo-rep/schedule_georep.py.in index 1887fe7ba87..48b2b507060 100644 --- a/extras/geo-rep/schedule_georep.py.in +++ b/extras/geo-rep/schedule_georep.py.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 """ Schedule Geo-replication ------------------------ @@ -43,7 +43,7 @@ SESSION_MOUNT_LOG_FILE = ("/var/log/glusterfs/geo-replication" "/schedule_georep.mount.log") USE_CLI_COLOR = True - +mnt_list = [] class GlusterBadXmlFormat(Exception): """ @@ -83,13 +83,15 @@ def execute(cmd, success_msg="", failure_msg="", exitcode=-1): On success it can print message in stdout if specified. On failure, exits after writing to stderr. """ - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = p.communicate() if p.returncode == 0: if success_msg: output_ok(success_msg) return out else: + if exitcode == 0: + return err_msg = err if err else out output_notok(failure_msg, err=err_msg, exitcode=exitcode) @@ -112,12 +114,12 @@ def cleanup(hostname, volname, mnt): """ Unmount the Volume and Remove the temporary directory """ - execute(["umount", mnt], + execute(["umount", "-l", mnt], failure_msg="Unable to Unmount Gluster Volume " "{0}:{1}(Mounted at {2})".format(hostname, volname, mnt)) execute(["rmdir", mnt], failure_msg="Unable to Remove temp directory " - "{0}".format(mnt)) + "{0}".format(mnt), exitcode=0) @contextmanager @@ -130,6 +132,7 @@ def glustermount(hostname, volname): Automatically unmounts it in case of Exceptions/out of context """ mnt = tempfile.mkdtemp(prefix="georepsetup_") + mnt_list.append(mnt) execute(["@SBIN_DIR@/glusterfs", "--volfile-server", hostname, "--volfile-id", volname, @@ -297,6 +300,7 @@ def get_summary(mastervol, slave_url): status_data = get(mastervol, slave_url) for session in status_data: + session_name = "" summary = { "active": 0, "passive": 0, @@ -339,7 +343,8 @@ def get_summary(mastervol, slave_url): if summary["faulty"] == 0 and summary["offline"] == 0: summary["ok"] = True - out.append([session_name, summary, faulty_rows, down_rows]) + if session_name != "": + out.append([session_name, summary, faulty_rows, down_rows]) return out @@ -347,7 +352,7 @@ def get_summary(mastervol, slave_url): def touch_mount_root(mastervol): # Create a Mount and Touch the Mount point root, # Hack to make sure some event available after - # setting Checkpoint. Without this their is a chance of + # setting Checkpoint. Without this there is a chance of # Checkpoint never completes. with glustermount("localhost", mastervol) as mnt: execute(["touch", mnt]) @@ -376,14 +381,14 @@ def main(args): output_ok("Started Geo-replication and watching Status for " "Checkpoint completion") - touch_mount_root(args.mastervol) - start_time = int(time.time()) duration = 0 # Sleep till Geo-rep initializes time.sleep(60) + touch_mount_root(args.mastervol) + slave_url = "{0}::{1}".format(args.slave, args.slavevol) # Loop to Check the Geo-replication Status and Checkpoint @@ -397,41 +402,39 @@ def main(args): # or any other error. Gluster cmd still produces XML output # with different message output_warning("Unable to get Geo-replication Status") - time.sleep(1) - continue - - session_name, summary, faulty_rows, down_rows = session_summary[0] - chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \ - "NOT COMPLETE" - ok_status = "OK" if summary["ok"] else "NOT OK" - - if summary["ok"]: - output_ok("All Checkpoints {1}, " - "All status {2} (Turns {0:>3})".format( - turns, chkpt_status, ok_status)) - else: - output_warning("All Checkpoints {1}, " - "All status {2} (Turns {0:>3})".format( - turns, chkpt_status, ok_status)) - - output_warning("Geo-rep workers Faulty/Offline, " - "Faulty: {0} Offline: {1}".format( - repr(faulty_rows), - repr(down_rows))) - - if summary["checkpoints_ok"]: - output_ok("Stopping Geo-replication session now") - cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", - args.mastervol, - "%s::%s" % (args.slave, args.slavevol), "stop"] - execute(cmd) - break else: - # If Checkpoint is not complete after a iteration means brick - # was down and came online now. SETATTR on mount is not - # recorded, So again issue touch on mount root So that - # Stime will increase and Checkpoint will complete. - touch_mount_root(args.mastervol) + session_name, summary, faulty_rows, down_rows = session_summary[0] + chkpt_status = "COMPLETE" if summary["checkpoints_ok"] else \ + "NOT COMPLETE" + ok_status = "OK" if summary["ok"] else "NOT OK" + + if summary["ok"]: + output_ok("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + else: + output_warning("All Checkpoints {1}, " + "All status {2} (Turns {0:>3})".format( + turns, chkpt_status, ok_status)) + + output_warning("Geo-rep workers Faulty/Offline, " + "Faulty: {0} Offline: {1}".format( + repr(faulty_rows), + repr(down_rows))) + + if summary["checkpoints_ok"]: + output_ok("Stopping Geo-replication session now") + cmd = ["@SBIN_DIR@/gluster", "volume", "geo-replication", + args.mastervol, + "%s::%s" % (args.slave, args.slavevol), "stop"] + execute(cmd) + break + else: + # If Checkpoint is not complete after a iteration means brick + # was down and came online now. SETATTR on mount is not + # recorded, So again issue touch on mount root So that + # Stime will increase and Checkpoint will complete. + touch_mount_root(args.mastervol) # Increment the turns and Sleep for 10 sec turns += 1 @@ -446,13 +449,18 @@ def main(args): time.sleep(args.interval) + for mnt in mnt_list: + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) + if __name__ == "__main__": parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, description=__doc__) parser.add_argument("mastervol", help="Master Volume Name") parser.add_argument("slave", - help="SLAVEHOST or root@SLAVEHOST " - "or user@SLAVEHOST", + help="Slave hostname " + "(<username>@SLAVEHOST or SLAVEHOST)", metavar="SLAVE") parser.add_argument("slavevol", help="Slave Volume Name") parser.add_argument("--interval", help="Interval in Seconds. " @@ -474,4 +482,11 @@ if __name__ == "__main__": execute(cmd) main(args) except KeyboardInterrupt: + for mnt in mnt_list: + execute(["umount", "-l", mnt], + failure_msg="Unable to Unmount Gluster Volume " + "Mounted at {0}".format(mnt), exitcode=0) + execute(["rmdir", mnt], + failure_msg="Unable to Remove temp directory " + "{0}".format(mnt), exitcode=0) output_notok("Exiting...") diff --git a/extras/git-branch-diff.py b/extras/git-branch-diff.py index c8d74ec9f31..382513e069e 100755 --- a/extras/git-branch-diff.py +++ b/extras/git-branch-diff.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/bin/python2 """ Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com> @@ -75,6 +75,7 @@ Prasanna Kumar Kalever <prasanna.kalever@redhat.com> """ +from __future__ import print_function import os import sys import argparse @@ -118,16 +119,16 @@ class GitBranchDiff: status_tbr, op = commands.getstatusoutput('git log ' + self.t_pattern) if status_sbr != 0: - print "Error: --source=" + self.s_pattern + " doesn't exit\n" + print("Error: --source=" + self.s_pattern + " doesn't exit\n") self.parser.print_help() exit(status_sbr) elif status_tbr != 0: - print "Error: --target=" + self.t_pattern + " doesn't exit\n" + print("Error: --target=" + self.t_pattern + " doesn't exit\n") self.parser.print_help() exit(status_tbr) def check_author_exist (self): - " defend to check given author exist, format incase of multiple" + " defend to check given author exist, format in case of multiple" contrib_list = ['', '*', 'all', 'All', 'ALL', 'null', 'Null', 'NULL'] if self.g_author in contrib_list: self.g_author = "" @@ -137,8 +138,8 @@ class GitBranchDiff: cmd4 = 'git log ' + self.s_pattern + ' --author=' + ide c_list = subprocess.check_output(cmd4, shell = True) if len(c_list) is 0: - print "Error: --author=%s doesn't exit" %self.g_author - print "see '%s --help'" %__file__ + print("Error: --author=%s doesn't exit" %self.g_author) + print("see '%s --help'" %__file__) exit(1) if len(ide_list) > 1: self.g_author = "\|".join(ide_list) @@ -150,16 +151,16 @@ class GitBranchDiff: return True except requests.Timeout as err: " request timed out" - print "Warning: failed to get list of open review commits on " \ + print("Warning: failed to get list of open review commits on " \ "gerrit.\n" \ "hint: Request timed out! gerrit server could possibly " \ - "slow ...\n" + "slow ...\n") return False except requests.RequestException as err: " handle other errors" - print "Warning: failed to get list of open review commits on " \ + print("Warning: failed to get list of open review commits on " \ "gerrit\n" \ - "hint: check with internet connection ...\n" + "hint: check with internet connection ...\n") return False def parse_cmd_args (self): @@ -189,7 +190,7 @@ class GitBranchDiff: '--author', help = 'default: git config name/email, ' 'to provide multiple specify comma' - ' seperated values', + ' separated values', default = author, dest = 'author') self.parser.add_argument('-p', @@ -212,39 +213,39 @@ class GitBranchDiff: def print_output (self): " display the result list" - print "\n------------------------------------------------------------\n" - print self.tick + " Successfully Backported changes:" - print ' {' + 'from: ' + self.s_pattern + \ - ' to: '+ self.t_pattern + '}\n' - for key, value in self.s_dict.iteritems(): + print("\n------------------------------------------------------------\n") + print(self.tick + " Successfully Backported changes:") + print(' {' + 'from: ' + self.s_pattern + \ + ' to: '+ self.t_pattern + '}\n') + for key, value in self.s_dict.items(): if value in self.t_dict.itervalues(): - print "[%s%s%s] %s" %(self.yello_set, + print("[%s%s%s] %s" %(self.yello_set, key, self.color_unset, - value) - print "\n------------------------------------------------------------\n" - print self.cross + " Missing patches in " + self.t_pattern + ':\n' + value)) + print("\n------------------------------------------------------------\n") + print(self.cross + " Missing patches in " + self.t_pattern + ':\n') if self.connected_to_gerrit(): cmd3 = "git review -r origin -l" review_list = subprocess.check_output(cmd3, shell = True).split('\n') else: review_list = [] - for key, value in self.s_dict.iteritems(): + for key, value in self.s_dict.items(): if value not in self.t_dict.itervalues(): if any(value in s for s in review_list): - print "[%s%s%s] %s %s(under review)%s" %(self.yello_set, + print("[%s%s%s] %s %s(under review)%s" %(self.yello_set, key, self.color_unset, value, self.green_set, - self.color_unset) + self.color_unset)) else: - print "[%s%s%s] %s" %(self.yello_set, + print("[%s%s%s] %s" %(self.yello_set, key, self.color_unset, - value) - print "\n------------------------------------------------------------\n" + value)) + print("\n------------------------------------------------------------\n") def main (self): self.check_pattern_exist() @@ -262,8 +263,8 @@ class GitBranchDiff: t_list = subprocess.check_output(cmd2, shell = True) if len(t_list) is 0: - print "No commits in the target: %s" %self.t_pattern - print "see '%s --help'" %__file__ + print("No commits in the target: %s" %self.t_pattern) + print("see '%s --help'" %__file__) exit() else: t_list = t_list.split('\n') diff --git a/extras/glusterd.vol.in b/extras/glusterd.vol.in index 5338aa25b0d..5d7bad0e4c8 100644 --- a/extras/glusterd.vol.in +++ b/extras/glusterd.vol.in @@ -1,13 +1,15 @@ volume management type mgmt/glusterd option working-directory @GLUSTERD_WORKDIR@ - option transport-type socket,rdma + option transport-type socket option transport.socket.keepalive-time 10 option transport.socket.keepalive-interval 2 option transport.socket.read-fail-log off + option transport.socket.listen-port 24007 option ping-timeout 0 option event-threads 1 # option lock-timer 180 # option transport.address-family inet6 # option base-port 49152 + option max-port 60999 end-volume diff --git a/extras/glusterfs-georep-logrotate b/extras/glusterfs-georep-logrotate index 6fdb8c65aaf..3e7ecf373a1 100644 --- a/extras/glusterfs-georep-logrotate +++ b/extras/glusterfs-georep-logrotate @@ -1,6 +1,12 @@ /var/log/glusterfs/geo-replication/*/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -15,7 +21,13 @@ /var/log/glusterfs/geo-replication-slaves/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -30,7 +42,13 @@ /var/log/glusterfs/geo-replication-slaves/*/*.log { sharedscripts - rotate 52 + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + missingok compress delaycompress diff --git a/extras/glusterfs-georep-upgrade.py b/extras/glusterfs-georep-upgrade.py new file mode 100755 index 00000000000..634576058d6 --- /dev/null +++ b/extras/glusterfs-georep-upgrade.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +""" + +Copyright (c) 2020 Red Hat, Inc. <http://www.redhat.com> +This file is part of GlusterFS. + +This file is licensed to you under your choice of the GNU Lesser +General Public License, version 3 or any later version (LGPLv3 or +later), or the GNU General Public License, version 2 (GPLv2), in all +cases as published by the Free Software Foundation. + +""" + +import argparse +import errno +import os, sys +import shutil +from datetime import datetime + +def find_htime_path(brick_path): + dirs = [] + htime_dir = os.path.join(brick_path, '.glusterfs/changelogs/htime') + for file in os.listdir(htime_dir): + if os.path.isfile(os.path.join(htime_dir,file)) and file.startswith("HTIME"): + dirs.append(os.path.join(htime_dir, file)) + else: + raise FileNotFoundError("%s unavailable" % (os.path.join(htime_dir, file))) + return dirs + +def modify_htime_file(brick_path): + htime_file_path_list = find_htime_path(brick_path) + + for htime_file_path in htime_file_path_list: + changelog_path = os.path.join(brick_path, '.glusterfs/changelogs') + temp_htime_path = os.path.join(changelog_path, 'htime/temp_htime_file') + with open(htime_file_path, 'r') as htime_file, open(temp_htime_path, 'w') as temp_htime_file: + #extract epoch times from htime file + paths = htime_file.read().split("\x00") + + for pth in paths: + epoch_no = pth.split(".")[-1] + changelog = os.path.basename(pth) + #convert epoch time to year, month and day + if epoch_no != '': + date=(datetime.fromtimestamp(float(int(epoch_no))).strftime("%Y/%m/%d")) + #update paths in temp htime file + temp_htime_file.write("%s/%s/%s\x00" % (changelog_path, date, changelog)) + #create directory in the format year/month/days + path = os.path.join(changelog_path, date) + + if changelog.startswith("CHANGELOG."): + try: + os.makedirs(path, mode = 0o600); + except OSError as exc: + if exc.errno == errno.EEXIST: + pass + else: + raise + + #copy existing changelogs to new directory structure, delete old changelog files + shutil.copyfile(pth, os.path.join(path, changelog)) + os.remove(pth) + + #rename temp_htime_file with htime file + os.rename(htime_file_path, os.path.join('%s.bak'%htime_file_path)) + os.rename(temp_htime_path, htime_file_path) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('brick_path', help="This upgrade script, which is to be run on\ + server side, takes brick path as the argument, \ + updates paths inside htime file and alters the directory structure \ + above the changelog files inorder to support new optimised format \ + of the directory structure as per \ + https://review.gluster.org/#/c/glusterfs/+/23733/") + args = parser.parse_args() + modify_htime_file(args.brick_path) diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate index 575c0eee771..6ba6ef18e9f 100644 --- a/extras/glusterfs-logrotate +++ b/extras/glusterfs-logrotate @@ -2,7 +2,12 @@ /var/log/glusterfs/*.log { sharedscripts weekly - rotate 52 + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -17,7 +22,12 @@ /var/log/glusterfs/bricks/*.log { sharedscripts weekly - rotate 52 + maxsize 10M + minsize 100k + +# 6 months of logs are good enough + rotate 26 + missingok compress delaycompress @@ -35,3 +45,24 @@ compress delaycompress } + +# Rotate snapd log +/var/log/glusterfs/snaps/*/*.log { + sharedscripts + weekly + maxsize 10M + minsize 100k + + # 6 months of logs are good enough + rotate 26 + + missingok + compress + delaycompress + notifempty + postrotate + for pid in `ps -aef | grep glusterfs | egrep "snapd" | awk '{print $2}'`; do + /usr/bin/kill -HUP $pid > /dev/null 2>&1 || true + done + endscript +} diff --git a/extras/gnfs-loganalyse.py b/extras/gnfs-loganalyse.py index 71e79b6be4e..6341d007188 100644..100755 --- a/extras/gnfs-loganalyse.py +++ b/extras/gnfs-loganalyse.py @@ -10,6 +10,7 @@ """ +from __future__ import print_function import os import string import sys @@ -72,7 +73,7 @@ class NFSRequest: self.replygfid = tokens [gfididx + 1].strip(",") def dump (self): - print "ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata + print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata) class NFSLogAnalyzer: @@ -149,7 +150,7 @@ class NFSLogAnalyzer: return rcount = len (self.xid_request_map.keys ()) orphancount = len (self.orphan_replies.keys ()) - print "Requests: " + str(rcount) + ", Orphans: " + str(orphancount) + print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount)) def dump (self): self.getStats () diff --git a/extras/group-db-workload b/extras/group-db-workload new file mode 100644 index 00000000000..9334d6fb942 --- /dev/null +++ b/extras/group-db-workload @@ -0,0 +1,12 @@ +performance.open-behind=on +performance.write-behind=off +performance.stat-prefetch=off +performance.quick-read=off +performance.strict-o-direct=on +performance.read-ahead=off +performance.io-cache=off +performance.readdir-ahead=off +performance.client-io-threads=on +server.event-threads=4 +client.event-threads=4 +performance.read-after-open=yes diff --git a/extras/group-distributed-virt b/extras/group-distributed-virt new file mode 100644 index 00000000000..a960b76c694 --- /dev/null +++ b/extras/group-distributed-virt @@ -0,0 +1,10 @@ +performance.quick-read=off +performance.read-ahead=off +performance.io-cache=off +performance.low-prio-threads=32 +network.remote-dio=enable +features.shard=on +user.cifs=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on diff --git a/extras/group-gluster-block b/extras/group-gluster-block index e94f834d420..1e398019e6b 100644 --- a/extras/group-gluster-block +++ b/extras/group-gluster-block @@ -5,8 +5,16 @@ performance.stat-prefetch=off performance.open-behind=off performance.readdir-ahead=off performance.strict-o-direct=on +performance.client-io-threads=on +performance.io-thread-count=32 +performance.high-prio-threads=32 +performance.normal-prio-threads=32 +performance.low-prio-threads=32 +performance.least-prio-threads=4 +client.event-threads=8 +server.event-threads=8 network.remote-dio=disable -cluster.eager-lock=disable +cluster.eager-lock=enable cluster.quorum-type=auto cluster.data-self-heal-algorithm=full cluster.locking-scheme=granular @@ -16,3 +24,4 @@ features.shard=on features.shard-block-size=64MB user.cifs=off server.allow-insecure=on +cluster.choose-local=off diff --git a/extras/group-metadata-cache b/extras/group-metadata-cache index 0f780eb91aa..b890b288fc7 100644 --- a/extras/group-metadata-cache +++ b/extras/group-metadata-cache @@ -3,4 +3,4 @@ features.cache-invalidation-timeout=600 performance.stat-prefetch=on performance.cache-invalidation=on performance.md-cache-timeout=600 -network.inode-lru-limit=50000 +network.inode-lru-limit=200000 diff --git a/extras/group-nl-cache b/extras/group-nl-cache index a41e8ecfd64..897807e8933 100644 --- a/extras/group-nl-cache +++ b/extras/group-nl-cache @@ -2,4 +2,4 @@ features.cache-invalidation=on features.cache-invalidation-timeout=600 performance.nl-cache=on performance.nl-cache-timeout=600 -network.inode-lru-limit=50000 +network.inode-lru-limit=200000 diff --git a/extras/group-samba b/extras/group-samba new file mode 100644 index 00000000000..eeee6e06031 --- /dev/null +++ b/extras/group-samba @@ -0,0 +1,11 @@ +features.cache-invalidation=on +features.cache-invalidation-timeout=600 +performance.cache-samba-metadata=on +performance.stat-prefetch=on +performance.cache-invalidation=on +performance.md-cache-timeout=600 +network.inode-lru-limit=200000 +performance.nl-cache=on +performance.nl-cache-timeout=600 +performance.readdir-ahead=on +performance.parallel-readdir=on diff --git a/extras/group-virt.example b/extras/group-virt.example index 403bed6714f..cc37c98a25c 100644 --- a/extras/group-virt.example +++ b/extras/group-virt.example @@ -2,7 +2,8 @@ performance.quick-read=off performance.read-ahead=off performance.io-cache=off performance.low-prio-threads=32 -network.remote-dio=enable +network.remote-dio=disable +performance.strict-o-direct=on cluster.eager-lock=enable cluster.quorum-type=auto cluster.server-quorum-type=server @@ -12,3 +13,12 @@ cluster.shd-max-threads=8 cluster.shd-wait-qlength=10000 features.shard=on user.cifs=off +cluster.choose-local=off +client.event-threads=4 +server.event-threads=4 +performance.client-io-threads=on +network.ping-timeout=20 +server.tcp-user-timeout=20 +server.keepalive-time=10 +server.keepalive-interval=2 +server.keepalive-count=5 diff --git a/extras/hook-scripts/S40ufo-stop.py b/extras/hook-scripts/S40ufo-stop.py index 107f1968355..2c79eb1d54a 100755 --- a/extras/hook-scripts/S40ufo-stop.py +++ b/extras/hook-scripts/S40ufo-stop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 import os from optparse import OptionParser diff --git a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh index a5e472e9267..7d6052315bb 100755 --- a/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh +++ b/extras/hook-scripts/S56glusterd-geo-rep-create-post.sh @@ -77,18 +77,28 @@ if [ "$val" == "" ]; then exit; fi SSH_PORT=`echo $val` +SSH_OPT="-oPasswordAuthentication=no -oStrictHostKeyChecking=no" if [ -f $pub_file ]; then # For a non-root user copy the pub file to the user's home directory # For a root user copy the pub files to priv_dir->geo-rep. if [ "$slave_user" != "root" ]; then - slave_user_home_dir=`ssh -p ${SSH_PORT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"` - scp -P ${SSH_PORT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp - ssh -p ${SSH_PORT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub" + slave_user_home_dir=`ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "getent passwd $slave_user | cut -d ':' -f 6"` + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_user@$slave_ip:$slave_user_home_dir/common_secret.pem.pub_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_user@$slave_ip "mv $slave_user_home_dir/common_secret.pem.pub_tmp $slave_user_home_dir/${mastervol}_${slavevol}_common_secret.pem.pub" else - scp -P ${SSH_PORT} $pub_file $slave_ip:$pub_file_tmp - ssh -p ${SSH_PORT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" - ssh -p ${SSH_PORT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" - ssh -p ${SSH_PORT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + if [[ -z "${GR_SSH_IDENTITY_KEY}" ]]; then + scp -P ${SSH_PORT} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + else + scp -P ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $pub_file $slave_ip:$pub_file_tmp + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "mv $pub_file_tmp ${pub_file_dname}/${mastervol}_${slavevol}_${pub_file_bname}" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: copy file /geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster system:: execute add_secret_pub root geo-replication/${mastervol}_${slavevol}_common_secret.pem.pub > /dev/null" + ssh -p ${SSH_PORT} -i ${GR_SSH_IDENTITY_KEY} ${SSH_OPT} $slave_ip "gluster vol set ${slavevol} features.read-only on" + fi fi fi diff --git a/extras/hook-scripts/add-brick/post/Makefile.am b/extras/hook-scripts/add-brick/post/Makefile.am index 5ca5a669de9..9b236df096d 100644 --- a/extras/hook-scripts/add-brick/post/Makefile.am +++ b/extras/hook-scripts/add-brick/post/Makefile.am @@ -1,4 +1,6 @@ -EXTRA_DIST = disabled-quota-root-xattr-heal.sh +EXTRA_DIST = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/post/ -hook_SCRIPTS = disabled-quota-root-xattr-heal.sh +if WITH_SERVER +hook_SCRIPTS = disabled-quota-root-xattr-heal.sh S10selinux-label-brick.sh S13create-subdir-mounts.sh +endif diff --git a/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh new file mode 100755 index 00000000000..4a17c993a77 --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S10selinux-label-brick.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Install to hooks/<HOOKS_VER>/add-brick/post +# +# Add an SELinux file context for each brick using the glusterd_brick_t type. +# This ensures that the brick is relabeled correctly on an SELinux restart or +# restore. Subsequently, run a restore on the brick path to set the selinux +# labels. +# +### + +PROGNAME="Sselinux" +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +VOL= + +parse_args () { + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift + done +} + +set_brick_labels() +{ + local volname="${1}" + local fctx + local list=() + + fctx="$(semanage fcontext --list -C)" + + # wait for new brick path to be updated under + # ${GLUSTERD_WORKDIR}/vols/${volname}/bricks/ + sleep 5 + + # grab the path for each local brick + brickpath="${GLUSTERD_WORKDIR}/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) + + # create a list of bricks for which custom SELinux + # label doesn't exist + for b in ${brickdirs}; do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -ne 0 ]]; then + list+=("${pattern}") + fi + done + + # Add a file context for each brick path in the list and associate with the + # glusterd_brick_t SELinux type. + for p in ${list[@]} + do + semanage fcontext --add -t glusterd_brick_t -r s0 "${p}" + done + + # Set the labels for which SELinux label was added above + for b in ${brickdirs} + do + echo "${list[@]}" | grep "${b}" >/dev/null + if [[ $? -eq 0 ]]; then + restorecon -R "${b}" + fi + done +} + +SELINUX_STATE=$(which getenforce && getenforce) +[ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 + +parse_args "$@" +[ -z "${VOL}" ] && exit 1 + +set_brick_labels "${VOL}" + +exit 0 diff --git a/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh new file mode 100755 index 00000000000..1a6923ee7aa --- /dev/null +++ b/extras/hook-scripts/add-brick/post/S13create-subdir-mounts.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +##--------------------------------------------------------------------------- +## This script runs the self-heal of the directories which are expected to +## be present as they are mounted as subdirectory mounts. +##--------------------------------------------------------------------------- + +MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`; +OPTSPEC="volname:,version:,gd-workdir:,volume-op:" +PROGNAME="add-brick-create-subdir" +VOL_NAME=test +GLUSTERD_WORKDIR="/var/lib/glusterd" + +cleanup_mountpoint () +{ + umount -f $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi + + rmdir $MOUNT_DIR; + if [ 0 -ne $? ] + then + return $? + fi +} + +##------------------------------------------ +## Parse the arguments +##------------------------------------------ +ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +eval set -- "$ARGS" + +while true; +do + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + ;; + --volume-op) + shift + ;; + *) + shift + break + ;; + esac + shift +done + +## See if we have any subdirs to be healed before going further +subdirs=$(grep 'auth.allow' ${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info | cut -f2 -d'=' | tr ',' '\n' | cut -f1 -d'('); + +if [ -z ${subdirs} ]; then + rmdir $MOUNT_DIR; + exit 0; +fi + +##---------------------------------------- +## Mount the volume in temp directory. +## ----------------------------------- +glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-50 $MOUNT_DIR; +if [ 0 -ne $? ] +then + exit $?; +fi + +## ----------------------------------- +# Do the 'stat' on all the directory for now. Ideal fix is to look at subdir +# list from 'auth.allow' option and only stat them. +for subdir in ${subdirs} +do + stat ${MOUNT_DIR}/${subdir} > /dev/null; +done + +## Clean up and exit +cleanup_mountpoint; diff --git a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh index bde7249d429..ca17a903549 100755 --- a/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh +++ b/extras/hook-scripts/add-brick/post/disabled-quota-root-xattr-heal.sh @@ -13,123 +13,133 @@ QUOTA_LIMIT_XATTR="trusted.glusterfs.quota.limit-set" QUOTA_OBJECT_LIMIT_XATTR="trusted.glusterfs.quota.limit-objects" -MOUNT_DIR=`mktemp -d -t ${0##*/}.XXXXXX`; +MOUNT_DIR=$(mktemp -d -t "${0##*/}.XXXXXX"); OPTSPEC="volname:,version:,gd-workdir:,volume-op:" PROGNAME="Quota-xattr-heal-add-brick" VOL_NAME= VERSION= VOLUME_OP= GLUSTERD_WORKDIR= -ENABLED_NAME="S28Quota-root-xattr-heal.sh" +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" + +THIS_SCRIPT=$(echo "${0}" | awk -F'/' '{print $NF}') cleanup_mountpoint () { - umount -f $MOUNT_DIR; - if [ 0 -ne $? ] - then - return $? - fi - - rmdir $MOUNT_DIR; - if [ 0 -ne $? ] - then - return $? - fi + + if umount -f "${MOUNT_DIR}"; then + return $? + fi + + if rmdir "${MOUNT_DIR}"; then + return $? + fi } disable_and_exit () { - if [ -e "$ENABLED_STATE" ] - then - unlink $ENABLED_STATE; - exit $? - fi + if [ -e "${ENABLED_STATE}" ] + then + unlink "${ENABLED_STATE}"; + exit $? + fi - exit 0 + exit 0 } get_and_set_xattr () { - XATTR=$1 - - VALUE=$(getfattr -n $XATTR -e hex --absolute-names $MOUNT_DIR 2>&1) - RET=$? - if [ 0 -eq $RET ]; then - VALUE=$(echo $VALUE | grep $XATTR | awk -F'=' '{print $NF}') - setfattr -n $XATTR -v $VALUE $MOUNT_DIR; - RET=$? - else - echo $VALUE | grep -iq "No such attribute" - if [ 0 -eq $? ]; then - RET=0 - fi - fi - - return $RET; + XATTR=$1 + + VALUE=$(getfattr -n "${XATTR}" -e hex --absolute-names "${MOUNT_DIR}" 2>&1) + RET=$? + if [ 0 -eq ${RET} ]; then + VALUE=$(echo "${VALUE}" | grep "${XATTR}" | awk -F'=' '{print $NF}') + setfattr -n "${XATTR}" -v "${VALUE}" "${MOUNT_DIR}"; + RET=$? + else + if echo "${VALUE}" | grep -iq "No such attribute" ; then + RET=0 + fi + fi + + return ${RET}; } ##------------------------------------------ ## Parse the arguments ##------------------------------------------ -ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") eval set -- "$ARGS" while true; do - case $1 in - --volname) - shift - VOL_NAME=$1 - ;; - --version) - shift - VERSION=$1 - ;; - --gd-workdir) - shift - GLUSTERD_WORKDIR=$1 - ;; - --volume-op) - shift - VOLUME_OP=$1 - ;; - *) - shift - break - ;; - esac - shift + case $1 in + --volname) + shift + VOL_NAME=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + *) + shift + break + ;; + esac + shift done ##---------------------------------------- -ENABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/$VOLUME_OP/post/$ENABLED_NAME" +# Avoid long lines +ENABLED_STATE_1="${GLUSTERD_WORKDIR}/hooks/${VERSION}/${VOLUME_OP}/" +ENABLED_STATE_2="post/${ENABLED_NAME_PREFIX}${VOL_NAME}-${ENABLED_NAME}" +ENABLED_STATE="${ENABLED_STATE_1}${ENABLED_STATE_2}" + +if [ "${THIS_SCRIPT}" != *"${VOL_NAME}"* ]; then + exit 0 +fi ## Is quota enabled? -FLAG=`grep "^features.quota=" $GLUSTERD_WORKDIR/vols/$VOL_NAME/info \ - | awk -F'=' '{print $NF}'`; -if [ "$FLAG" != "on" ] +FLAG=$(grep "^features.quota=" "${GLUSTERD_WORKDIR}/vols/${VOL_NAME}/info" \ +| awk -F'=' '{print $NF}'); +if [ "${FLAG}" != "on" ] then - disable_and_exit + disable_and_exit fi ## ----------------------------------- ## Mount the volume in temp directory. ## ----------------------------------- -glusterfs -s localhost --volfile-id=$VOL_NAME --client-pid=-42 $MOUNT_DIR; -if [ 0 -ne $? ] +# Avoid long lines +CMD_1="glusterfs -s localhost" +CMD_2="--volfile-id=${VOL_NAME} client-pid=-42 ${MOUNT_DIR}" +CMD="${CMD_1}${CMD_2}" + +if ${CMD} then - exit $?; + exit $?; fi ## ----------------------------------- -RET1=$(get_and_set_xattr $QUOTA_LIMIT_XATTR) -RET2=$(get_and_set_xattr $QUOTA_OBJECT_LIMIT_XATTR) +RET1=$(get_and_set_xattr "${QUOTA_LIMIT_XATTR}") +RET2=$(get_and_set_xattr "${QUOTA_OBJECT_LIMIT_XATTR}") ## Clean up and exit cleanup_mountpoint; -if [ $RET1 -ne 0 -o $RET2 -ne 0 ]; then - exit 1 +if [ "${RET1}" -ne 0 ] || [ "${RET2}" -ne 0 ]; then + exit 1 fi disable_and_exit; diff --git a/extras/hook-scripts/add-brick/pre/Makefile.am b/extras/hook-scripts/add-brick/pre/Makefile.am index 6329ad1d4bd..3288581aa57 100644 --- a/extras/hook-scripts/add-brick/pre/Makefile.am +++ b/extras/hook-scripts/add-brick/pre/Makefile.am @@ -1,4 +1,6 @@ EXTRA_DIST = S28Quota-enable-root-xattr-heal.sh hookdir = $(GLUSTERD_WORKDIR)/hooks/1/add-brick/pre/ +if WITH_SERVER hook_SCRIPTS = S28Quota-enable-root-xattr-heal.sh +endif diff --git a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh index 348f34ec3db..27e85231f45 100755 --- a/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh +++ b/extras/hook-scripts/add-brick/pre/S28Quota-enable-root-xattr-heal.sh @@ -26,10 +26,11 @@ VOL_NAME= GLUSTERD_WORKDIR= VOLUME_OP= VERSION= -ENABLED_NAME="S28Quota-root-xattr-heal.sh" +ENABLED_NAME_PREFIX="S28" +ENABLED_NAME="Quota-root-xattr-heal.sh" DISABLED_NAME="disabled-quota-root-xattr-heal.sh" -enable () +activate () { ln -sf $DISABLED_STATE $1; } @@ -37,7 +38,7 @@ enable () ##------------------------------------------ ## Parse the arguments ##------------------------------------------ -ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) +ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; @@ -69,8 +70,8 @@ done ##---------------------------------------- DISABLED_STATE="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$DISABLED_NAME" -ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/$ENABLED_NAME" -ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/$ENABLED_NAME"; +ENABLED_STATE_START="$GLUSTERD_WORKDIR/hooks/$VERSION/start/post/""$ENABLED_NAME_PREFIX$VOL_NAME""-""$ENABLED_NAME" +ENABLED_STATE_ADD_BRICK="$GLUSTERD_WORKDIR/hooks/$VERSION/add-brick/post/""$ENABLED_NAME_PREFIX""$VOL_NAME""-""$ENABLED_NAME"; ## Why to proceed if the required script itself is not present? ls $DISABLED_STATE; @@ -92,9 +93,9 @@ FLAG=`cat $GLUSTERD_WORKDIR/vols/$VOL_NAME/info | grep "^status=" \ | awk -F'=' '{print $NF}'`; if [ "$FLAG" != "1" ] then - enable $ENABLED_STATE_START; + activate $ENABLED_STATE_START; exit $? fi -enable $ENABLED_STATE_ADD_BRICK; +activate $ENABLED_STATE_ADD_BRICK; exit $? diff --git a/extras/hook-scripts/create/post/Makefile.am b/extras/hook-scripts/create/post/Makefile.am index adbce78d249..fd1892e9589 100644 --- a/extras/hook-scripts/create/post/Makefile.am +++ b/extras/hook-scripts/create/post/Makefile.am @@ -1,6 +1,8 @@ EXTRA_DIST = S10selinux-label-brick.sh scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/create/post/ +if WITH_SERVER if USE_SELINUX scripts_SCRIPTS = S10selinux-label-brick.sh endif +endif diff --git a/extras/hook-scripts/create/post/S10selinux-label-brick.sh b/extras/hook-scripts/create/post/S10selinux-label-brick.sh index a727226a492..f9b4b1a57e3 100755 --- a/extras/hook-scripts/create/post/S10selinux-label-brick.sh +++ b/extras/hook-scripts/create/post/S10selinux-label-brick.sh @@ -34,18 +34,21 @@ parse_args () { set_brick_labels() { - volname=${1} + volname="${1}" # grab the path for each local brick - brickpath="/var/lib/glusterd/vols/${volname}/bricks/*" - brickdirs=$(grep '^path=' "${brickpath}" | cut -d= -f 2 | sort -u) + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$( + find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | \ + sort -u + ) for b in ${brickdirs}; do # Add a file context for each brick path and associate with the # glusterd_brick_t SELinux type. - pattern="${b}\(/.*\)?" + pattern="${b}(/.*)?" semanage fcontext --add -t glusterd_brick_t -r s0 "${pattern}" - # Set the labels on the new brick path. restorecon -R "${b}" done @@ -54,7 +57,7 @@ set_brick_labels() SELINUX_STATE=$(which getenforce && getenforce) [ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 -parse_args $@ +parse_args "$@" [ -z "${VOL}" ] && exit 1 set_brick_labels "${VOL}" diff --git a/extras/hook-scripts/delete/pre/Makefile.am b/extras/hook-scripts/delete/pre/Makefile.am index bf0eabe255c..4fbfbe7311f 100644 --- a/extras/hook-scripts/delete/pre/Makefile.am +++ b/extras/hook-scripts/delete/pre/Makefile.am @@ -1,6 +1,8 @@ EXTRA_DIST = S10selinux-del-fcontext.sh scriptsdir = $(GLUSTERD_WORKDIR)/hooks/1/delete/pre/ +if WITH_SERVER if USE_SELINUX scripts_SCRIPTS = S10selinux-del-fcontext.sh endif +endif diff --git a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh index 2c83331d5cd..056b52afe76 100755 --- a/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh +++ b/extras/hook-scripts/delete/pre/S10selinux-del-fcontext.sh @@ -13,50 +13,61 @@ PROGNAME="Sselinux" OPTSPEC="volname:" VOL= -CONFIGFILE= -LOGFILEBASE= -PIDDIR= function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) - eval set -- "$ARGS" - - while true; do - case $1 in - --volname) - shift - VOL=$1 - ;; - *) - shift - break - ;; - esac + ARGS=$(getopt -o '' -l ${OPTSPEC} -n ${PROGNAME} -- "$@") + eval set -- "${ARGS}" + + while true; do + case ${1} in + --volname) + shift + VOL=${1} + ;; + *) shift - done + break + ;; + esac + shift + done } function delete_brick_fcontext() { - volname=$1 - - # grab the path for each local brick - brickdirs=$(grep '^path=' /var/lib/glusterd/vols/${volname}/bricks/* | cut -d= -f 2) + local volname=$1 + local fctx + local list=() - for b in $brickdirs - do - # remove the file context associated with the brick path - semanage fcontext --delete $b\(/.*\)? - done + fctx="$(semanage fcontext --list -C)" + # grab the path for each local brick + brickpath="/var/lib/glusterd/vols/${volname}/bricks/" + brickdirs=$(find "${brickpath}" -type f -exec grep '^path=' {} \; | \ + cut -d= -f 2 | sort -u) + for b in ${brickdirs} + do + pattern="${b}(/.*)?" + echo "${fctx}" | grep "^${pattern}\s" >/dev/null + if [[ $? -eq 0 ]]; then + list+=("${pattern}") + fi + done + if [[ ${#list[@]} -gt 0 ]]; then + printf 'fcontext --delete %s\n' "${list[@]}" | semanage -i - + fi + for b in ${brickdirs} + do + restorecon -R "${b}" + done } SELINUX_STATE=$(which getenforce && getenforce) [ "${SELINUX_STATE}" = 'Disabled' ] && exit 0 -parse_args $@ -[ -z "$VOL" ] && exit 1 +parse_args "$@" +[ -z "${VOL}" ] && exit 1 -delete_brick_fcontext $VOL +delete_brick_fcontext "${VOL}" # failure to delete the fcontext is not fatal exit 0 diff --git a/extras/hook-scripts/set/post/Makefile.am b/extras/hook-scripts/set/post/Makefile.am index cea579cb2d9..506a25a8666 100644 --- a/extras/hook-scripts/set/post/Makefile.am +++ b/extras/hook-scripts/set/post/Makefile.am @@ -1,4 +1,6 @@ EXTRA_DIST = S30samba-set.sh S32gluster_enable_shared_storage.sh hookdir = $(GLUSTERD_WORKDIR)/hooks/1/set/post/ +if WITH_SERVER hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/set/post/S30samba-set.sh b/extras/hook-scripts/set/post/S30samba-set.sh index 97d067fc33f..854f131f6c8 100755 --- a/extras/hook-scripts/set/post/S30samba-set.sh +++ b/extras/hook-scripts/set/post/S30samba-set.sh @@ -28,7 +28,7 @@ USERSMB_SET="" USERCIFS_SET="" function parse_args () { - ARGS=$(getopt -l $OPTSPEC --name $PROGNAME -o "o:" -- $@) + ARGS=$(getopt -o 'o:' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -89,7 +89,7 @@ function add_samba_share () { STRING+="glusterfs:loglevel = 7\n" STRING+="path = /\n" STRING+="read only = no\n" - STRING+="guest ok = yes\n" + STRING+="kernel share modes = no\n" printf "$STRING" >> ${CONFIGFILE} } @@ -103,9 +103,9 @@ function sighup_samba () { fi } -function del_samba_share () { +function deactivate_samba_share () { volname=$1 - sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE} + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} } function is_volume_started () { @@ -123,29 +123,39 @@ function get_smb () { usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\ cut -d"=" -f2) - if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then - uservalue="disable" + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "disable" ] || [ "$usercifsvalue" = "off" ]; then + uservalue="disable" + fi fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "disable" ] || [ "$usersmbvalue" = "off" ]; then + uservalue="disable" + fi + fi + echo "$uservalue" } -parse_args $@ -if [ "0" = $(is_volume_started "$VOL") ]; then +parse_args "$@" +if [ "0" = "$(is_volume_started "$VOL")" ]; then exit 0 fi -if [[ "$USERCIFS_SET" = "YES" || "$USERSMB_SET" = "YES" ]]; then +if [ "$USERCIFS_SET" = "YES" ] || [ "$USERSMB_SET" = "YES" ]; then #Find smb.conf, smbd pid directory and smbd logfile path find_config_info - if [ $(get_smb "$VOL") = "disable" ]; then - del_samba_share $VOL - sighup_samba + if [ "$(get_smb "$VOL")" = "disable" ]; then + deactivate_samba_share $VOL else if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then add_samba_share $VOL - sighup_samba + else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' ${CONFIGFILE} fi fi + sighup_samba fi diff --git a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh index c0aa73548fd..1f2564b44ff 100755 --- a/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh +++ b/extras/hook-scripts/set/post/S32gluster_enable_shared_storage.sh @@ -79,9 +79,9 @@ done if [ "$option" == "disable" ]; then # Unmount the volume on all the nodes - umount /var/run/gluster/shared_storage - cat /etc/fstab | grep -v "gluster_shared_storage /var/run/gluster/shared_storage/" > /var/run/gluster/fstab.tmp - mv /var/run/gluster/fstab.tmp /etc/fstab + umount /run/gluster/shared_storage + cat /etc/fstab | grep -v "gluster_shared_storage /run/gluster/shared_storage/" > /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab fi if [ "$is_originator" == 1 ]; then @@ -104,8 +104,15 @@ function check_volume_status() echo $status } -mount_cmd="mount -t glusterfs "$local_node_hostname":/gluster_shared_storage \ - /var/run/gluster/shared_storage" +key=`echo $5 | cut -d '=' -f 1` +val=`echo $5 | cut -d '=' -f 2` +if [ "$key" == "transport.address-family" ]; then + mount_cmd="mount -t glusterfs -o xlator-option=transport.address-family=inet6 \ + $local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage" +else + mount_cmd="mount -t glusterfs $local_node_hostname:/gluster_shared_storage \ + /run/gluster/shared_storage" +fi if [ "$option" == "enable" ]; then retry=0; @@ -117,13 +124,13 @@ if [ "$option" == "enable" ]; then if [ "$retry" == 3 ]; then break; fi - status = check_volume_status; + status=$(check_volume_status) done # Mount the volume on all the nodes - umount /var/run/gluster/shared_storage - mkdir -p /var/run/gluster/shared_storage + umount /run/gluster/shared_storage + mkdir -p /run/gluster/shared_storage $mount_cmd - cp /etc/fstab /var/run/gluster/fstab.tmp - echo "$local_node_hostname:/gluster_shared_storage /var/run/gluster/shared_storage/ glusterfs defaults 0 0" >> /var/run/gluster/fstab.tmp - mv /var/run/gluster/fstab.tmp /etc/fstab + cp /etc/fstab /run/gluster/fstab.tmp + echo "$local_node_hostname:/gluster_shared_storage /run/gluster/shared_storage/ glusterfs defaults 0 0" >> /run/gluster/fstab.tmp + mv /run/gluster/fstab.tmp /etc/fstab fi diff --git a/extras/hook-scripts/start/post/Makefile.am b/extras/hook-scripts/start/post/Makefile.am index 384a5822a0c..792019d3c9f 100644 --- a/extras/hook-scripts/start/post/Makefile.am +++ b/extras/hook-scripts/start/post/Makefile.am @@ -1,4 +1,6 @@ -EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh +EXTRA_DIST = S29CTDBsetup.sh S30samba-start.sh S31ganesha-start.sh hookdir = $(GLUSTERD_WORKDIR)/hooks/1/start/post/ +if WITH_SERVER hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/start/post/S29CTDBsetup.sh b/extras/hook-scripts/start/post/S29CTDBsetup.sh index 4265cba54ee..69a0d89a3eb 100755 --- a/extras/hook-scripts/start/post/S29CTDBsetup.sh +++ b/extras/hook-scripts/start/post/S29CTDBsetup.sh @@ -9,10 +9,14 @@ CTDB_MNT=/gluster/lock # Make sure ping-timeout is not default for CTDB volume PING_TIMEOUT_SECS=10 PROGNAME="ctdb" -OPTSPEC="volname:" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" HOSTNAME=`hostname` MNTOPTS="_netdev,transport=tcp,xlator-option=*client*.ping-timeout=${PING_TIMEOUT_SECS}" VOL= +GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= # $META is the volume that will be used by CTDB as a shared filesystem. # It is not desirable to use this volume for storing 'data' as well. # META is set to 'all' (viz. a keyword and hence not a legal volume name) @@ -21,7 +25,7 @@ VOL= META="all" function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -29,13 +33,27 @@ function parse_args () { --volname) shift VOL=$1 - ;; - + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; *) - shift - break - ;; - + shift + break + ;; esac shift @@ -55,7 +73,7 @@ function add_fstab_entry () { fi } -parse_args $@ +parse_args "$@" if [ "$META" = "$VOL" ] then mkdir -p $CTDB_MNT diff --git a/extras/hook-scripts/start/post/S30samba-start.sh b/extras/hook-scripts/start/post/S30samba-start.sh index 3e0f2573b60..cac0cbf1464 100755 --- a/extras/hook-scripts/start/post/S30samba-start.sh +++ b/extras/hook-scripts/start/post/S30samba-start.sh @@ -21,15 +21,18 @@ #volume. PROGNAME="Ssamba-start" -OPTSPEC="volname:,gd-workdir:" +OPTSPEC="volname:,gd-workdir:,version:,volume-op:,first:" VOL= CONFIGFILE= LOGFILEBASE= PIDDIR= GLUSTERD_WORKDIR= +VERSION= +VOLUME_OP= +FIRST= function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -42,24 +45,37 @@ function parse_args () { shift GLUSTERD_WORKDIR=$1 ;; + --version) + shift + VERSION=$1 + ;; + --volume-op) + shift + VOLUME_OP=$1 + ;; + --first) + shift + FIRST=$1 + ;; *) shift break ;; esac + shift done } function find_config_info () { - cmdout=`smbd -b | grep smb.conf` - if [ $? -ne 0 ];then + cmdout=$(smbd -b 2> /dev/null) + CONFIGFILE=$(echo "$cmdout" | grep CONFIGFILE | awk '{print $2}') + if [ -z "$CONFIGFILE" ]; then echo "Samba is not installed" exit 1 fi - CONFIGFILE=`echo $cmdout | awk {'print $2'}` - PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}` - LOGFILEBASE=`smbd -b | grep 'LOGFILEBASE' | awk '{print $2}'` + PIDDIR=$(echo "$cmdout" | grep PIDDIR | awk '{print $2}') + LOGFILEBASE=$(echo "$cmdout" | grep 'LOGFILEBASE' | awk '{print $2}') } function add_samba_share () { @@ -72,12 +88,12 @@ function add_samba_share () { STRING+="glusterfs:loglevel = 7\n" STRING+="path = /\n" STRING+="read only = no\n" - STRING+="guest ok = yes\n" - printf "$STRING" >> ${CONFIGFILE} + STRING+="kernel share modes = no\n" + printf "$STRING" >> "${CONFIGFILE}" } function sighup_samba () { - pid=`cat ${PIDDIR}/smbd.pid` + pid=$(cat "${PIDDIR}/smbd.pid" 2> /dev/null) if [ "x$pid" != "x" ] then kill -HUP "$pid"; @@ -90,26 +106,40 @@ function get_smb () { volname=$1 uservalue= - usercifsvalue=$(grep user.cifs $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + usercifsvalue=$(grep user.cifs "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ cut -d"=" -f2) - usersmbvalue=$(grep user.smb $GLUSTERD_WORKDIR/vols/"$volname"/info |\ + usersmbvalue=$(grep user.smb "$GLUSTERD_WORKDIR"/vols/"$volname"/info |\ cut -d"=" -f2) - if [[ $usercifsvalue = "disable" || $usersmbvalue = "disable" ]]; then - uservalue="disable" + if [ -n "$usercifsvalue" ]; then + if [ "$usercifsvalue" = "enable" ] || [ "$usercifsvalue" = "on" ]; then + uservalue="enable" + fi + fi + + if [ -n "$usersmbvalue" ]; then + if [ "$usersmbvalue" = "enable" ] || [ "$usersmbvalue" = "on" ]; then + uservalue="enable" + fi fi + echo "$uservalue" } -parse_args $@ -if [ "$(get_smb "$VOL")" = "disable" ]; then +parse_args "$@" + +value=$(get_smb "$VOL") + +if [ -z "$value" ] || [ "$value" != "enable" ]; then exit 0 fi #Find smb.conf, smbd pid directory and smbd logfile path find_config_info -if ! grep --quiet "\[gluster-$VOL\]" ${CONFIGFILE} ; then - add_samba_share $VOL - sighup_samba +if ! grep --quiet "\[gluster-$VOL\]" "${CONFIGFILE}" ; then + add_samba_share "$VOL" +else + sed -i '/\[gluster-'"$VOL"'\]/,/^$/!b;/available = no/d' "${CONFIGFILE}" fi +sighup_samba diff --git a/extras/hook-scripts/start/post/S31ganesha-start.sh b/extras/hook-scripts/start/post/S31ganesha-start.sh new file mode 100755 index 00000000000..7ad6f23ad06 --- /dev/null +++ b/extras/hook-scripts/start/post/S31ganesha-start.sh @@ -0,0 +1,122 @@ +#!/bin/bash +PROGNAME="Sganesha-start" +OPTSPEC="volname:,gd-workdir:" +VOL= +declare -i EXPORT_ID +ganesha_key="ganesha.enable" +GANESHA_DIR="/run/gluster/shared_storage/nfs-ganesha" +CONF1="$GANESHA_DIR/ganesha.conf" +GLUSTERD_WORKDIR= + +function parse_args () +{ + ARGS=$(getopt -l $OPTSPEC -o "o" -name $PROGNAME $@) + eval set -- "$ARGS" + + while true; do + case $1 in + --volname) + shift + VOL=$1 + ;; + --gd-workdir) + shift + GLUSTERD_WORKDIR=$1 + ;; + *) + shift + break + ;; + esac + shift + done +} + + + +#This function generates a new export entry as export.volume_name.conf +function write_conf() +{ +echo -e "# WARNING : Using Gluster CLI will overwrite manual +# changes made to this file. To avoid it, edit the +# file, copy it over to all the NFS-Ganesha nodes +# and run ganesha-ha.sh --refresh-config." + +echo "EXPORT{" +echo " Export_Id = 2;" +echo " Path = \"/$VOL\";" +echo " FSAL {" +echo " name = \"GLUSTER\";" +echo " hostname=\"localhost\";" +echo " volume=\"$VOL\";" +echo " }" +echo " Access_type = RW;" +echo " Disable_ACL = true;" +echo " Squash=\"No_root_squash\";" +echo " Pseudo=\"/$VOL\";" +echo " Protocols = \"3\", \"4\" ;" +echo " Transports = \"UDP\",\"TCP\";" +echo " SecType = \"sys\";" +echo "}" +} + +#It adds the export dynamically by sending dbus signals +function export_add() +{ + dbus-send --print-reply --system --dest=org.ganesha.nfsd \ +/org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.AddExport \ +string:$GANESHA_DIR/exports/export.$VOL.conf string:"EXPORT(Export_Id=$EXPORT_ID)" + +} + +# based on src/scripts/ganeshactl/Ganesha/export_mgr.py +function is_exported() +{ + local volume="${1}" + + dbus-send --type=method_call --print-reply --system \ + --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \ + org.ganesha.nfsd.exportmgr.ShowExports \ + | grep -w -q "/${volume}" + + return $? +} + +# Check the info file (contains the volume options) to see if Ganesha is +# enabled for this volume. +function ganesha_enabled() +{ + local volume="${1}" + local info_file="${GLUSTERD_WORKDIR}/vols/${VOL}/info" + local enabled="off" + + enabled=$(grep -w ${ganesha_key} ${info_file} | cut -d"=" -f2) + + [ "${enabled}" == "on" ] + + return $? +} + +parse_args $@ + +if ganesha_enabled ${VOL} && ! is_exported ${VOL} +then + if [ ! -e ${GANESHA_DIR}/exports/export.${VOL}.conf ] + then + #Remove export entry from nfs-ganesha.conf + sed -i /$VOL.conf/d $CONF1 + write_conf ${VOL} > ${GANESHA_DIR}/exports/export.${VOL}.conf + EXPORT_ID=`cat $GANESHA_DIR/.export_added` + EXPORT_ID=EXPORT_ID+1 + echo $EXPORT_ID > $GANESHA_DIR/.export_added + sed -i s/Export_Id.*/"Export_Id=$EXPORT_ID;"/ \ + $GANESHA_DIR/exports/export.$VOL.conf + echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF1 + else + EXPORT_ID=$(grep ^[[:space:]]*Export_Id $GANESHA_DIR/exports/export.$VOL.conf |\ + awk -F"[=,;]" '{print $2}' | tr -d '[[:space:]]') + fi + export_add $VOL +fi + +exit 0 diff --git a/extras/hook-scripts/stop/pre/Makefile.am b/extras/hook-scripts/stop/pre/Makefile.am index bf63e7393d3..9e8d1565e93 100644 --- a/extras/hook-scripts/stop/pre/Makefile.am +++ b/extras/hook-scripts/stop/pre/Makefile.am @@ -1,4 +1,6 @@ EXTRA_DIST = S29CTDB-teardown.sh S30samba-stop.sh hookdir = $(GLUSTERD_WORKDIR)/hooks/1/stop/pre/ +if WITH_SERVER hook_SCRIPTS = $(EXTRA_DIST) +endif diff --git a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh index 5fb49bd9e97..0975a00f18d 100755 --- a/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh +++ b/extras/hook-scripts/stop/pre/S29CTDB-teardown.sh @@ -2,8 +2,9 @@ CTDB_MNT=/gluster/lock PROGNAME="ctdb" -OPTSPEC="volname:" +OPTSPEC="volname:,last:" VOL= +LAST= # $META is the volume that will be used by CTDB as a shared filesystem. # It is not desirable to use this volume for storing 'data' as well. # META is set to 'all' (viz. a keyword and hence not a legal volume name) @@ -12,7 +13,7 @@ VOL= META="all" function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do @@ -20,15 +21,16 @@ function parse_args () { --volname) shift VOL=$1 - ;; - + ;; + --last) + shift + LAST=$1 + ;; *) - shift - break - ;; - + shift + break + ;; esac - shift done } @@ -51,7 +53,7 @@ function remove_fstab_entry () { fi } -parse_args $@ +parse_args "$@" if [ "$META" = "$VOL" ] then umount "$CTDB_MNT" diff --git a/extras/hook-scripts/stop/pre/S30samba-stop.sh b/extras/hook-scripts/stop/pre/S30samba-stop.sh index 62cf7d1e0d2..ea799381d62 100755 --- a/extras/hook-scripts/stop/pre/S30samba-stop.sh +++ b/extras/hook-scripts/stop/pre/S30samba-stop.sh @@ -16,27 +16,33 @@ #event by removing the volume related entries(if any) in smb.conf file. PROGNAME="Ssamba-stop" -OPTSPEC="volname:" +OPTSPEC="volname:,last:" VOL= CONFIGFILE= PIDDIR= +LAST= function parse_args () { - ARGS=$(getopt -l $OPTSPEC -name $PROGNAME $@) + ARGS=$(getopt -o '' -l $OPTSPEC -n $PROGNAME -- "$@") eval set -- "$ARGS" while true; do - case $1 in - --volname) - shift - VOL=$1 - ;; - *) - shift - break - ;; - esac - shift + case $1 in + --volname) + shift + VOL=$1 + ;; + --last) + shift + LAST=$1 + ;; + *) + shift + break + ;; + esac + + shift done } @@ -46,13 +52,13 @@ function find_config_info () { echo "Samba is not installed" exit 1 fi - CONFIGFILE=`echo $cmdout | awk {'print $2'}` - PIDDIR=`smbd -b | grep PIDDIR | awk {'print $2'}` + CONFIGFILE=`echo $cmdout | awk '{print $2}'` + PIDDIR=`smbd -b | grep PIDDIR | awk '{print $2}'` } -function del_samba_share () { +function deactivate_samba_share () { volname=$1 - sed -i "/\[gluster-$volname\]/,/^$/d" ${CONFIGFILE} + sed -i -e '/^\[gluster-'"$volname"'\]/{ :a' -e 'n; /available = no/H; /^$/!{$!ba;}; x; /./!{ s/^/available = no/; $!{G;x}; $H; }; s/.*//; x; };' ${CONFIGFILE} } function sighup_samba () { @@ -65,7 +71,7 @@ function sighup_samba () { fi } -parse_args $@ +parse_args "$@" find_config_info -del_samba_share $VOL +deactivate_samba_share $VOL sighup_samba diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh new file mode 100755 index 00000000000..ebc6bf144aa --- /dev/null +++ b/extras/identify-hangs.sh @@ -0,0 +1,53 @@ +#!/bin/bash +function get_statedump_fnames_without_timestamps +{ + ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u +} + +function get_non_uniq_fields +{ + local statedump_fname_prefix=$1 + print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}' +} + +function print_stack_lkowner_unique_in_one_line +{ + local statedump_fname_prefix=$1 + sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - - +} + +function get_stacks_that_appear_in_multiple_statedumps +{ + #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps + #print the stack + local statedump_fname_prefix=$1 + while read -r non_uniq_stack; + do + if [ -z "$printed" ]; + then + printed="1" + fi + echo "$statedump_fname_prefix" "$non_uniq_stack" + done < <(get_non_uniq_fields "$statedump_fname_prefix") +} + +statedumpdir=${1} +if [ -z "$statedumpdir" ]; +then + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +if [ ! -d "$statedumpdir" ]; +then + echo "$statedumpdir: Is not a directory" + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +cd "$statedumpdir" || exit 1 +for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps); +do + get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix" +done | column -t +echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2 diff --git a/extras/init.d/Makefile.am b/extras/init.d/Makefile.am index a9ac3b42d84..8d8cc69571a 100644 --- a/extras/init.d/Makefile.am +++ b/extras/init.d/Makefile.am @@ -1,7 +1,7 @@ -EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat glusterd-SuSE \ - glusterd.plist rhel5-load-fuse.modules \ - glustereventsd-FreeBSD glustereventsd-Redhat glustereventsd-Debian +EXTRA_DIST = glusterd-Debian glusterd-FreeBSD glusterd-Redhat \ + glusterd-SuSE glusterd.plist glustereventsd-FreeBSD \ + glustereventsd-Redhat glustereventsd-Debian CLEANFILES = @@ -10,11 +10,12 @@ SYSTEMD_DIR = @systemddir@ LAUNCHD_DIR = @launchddir@ $(GF_DISTRIBUTION): +if WITH_SERVER @if [ ! -d $(SYSTEMD_DIR) ]; then \ $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ $(INSTALL_PROGRAM) glusterd-$(GF_DISTRIBUTION) $(DESTDIR)$(INIT_DIR)/glusterd; \ fi - +endif if BUILD_EVENTS @if [ ! -d $(SYSTEMD_DIR) ]; then \ $(mkdir_p) $(DESTDIR)$(INIT_DIR); \ diff --git a/extras/init.d/rhel5-load-fuse.modules b/extras/init.d/rhel5-load-fuse.modules deleted file mode 100755 index ee194db99b8..00000000000 --- a/extras/init.d/rhel5-load-fuse.modules +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -# -# fusermount-glusterfs requires the /dev/fuse character device. The fuse module -# provides this and is loaded on demand in newer Linux distributions. -# - -[ -c /dev/fuse ] || /sbin/modprobe fuse diff --git a/extras/mount-shared-storage.sh b/extras/mount-shared-storage.sh index 3645a0f42fd..cc40e13c3e3 100755 --- a/extras/mount-shared-storage.sh +++ b/extras/mount-shared-storage.sh @@ -2,35 +2,38 @@ #Post reboot there is a chance in which mounting of shared storage will fail #This will impact starting of features like NFS-Ganesha. So this script will #try to mount the shared storage if it fails -#TODO : Do it for other glusterfs clients in /etc/fstab -volume="gluster_shared_storage" -mp="/var/run/gluster/shared_storage" -#check if there is fstab entry for shared storage -gfc=$(sed -e 's/#.$//' </etc/fstab | grep -c $volume) -if [ $gfc -eq 0 ] -then - exit 0 -fi +exitStatus=0 -#check whether shared storage is mounted -#if it is mounted then mount has inode value 1 -inode=$(ls -id $mp | awk '{print $1}') +while IFS= read -r glm +do + IFS=$' \t' read -r -a arr <<< "$glm" -if [ $inode -eq 1 ] -then - exit 0 -fi + #Validate storage type is glusterfs + if [ "${arr[2]}" == "glusterfs" ] + then -mount -t glusterfs localhost:/$volume $mp -#wait for few seconds -sleep 5 + #check whether shared storage is mounted + #if it is mounted then mountpoint -q will return a 0 success code + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} is already mounted" + continue + fi -#recheck mount got succeed -inode=$(ls -id $mp | awk '{print $1}') -if [ $inode -eq 1 ] -then - exit 0 -else - exit 1 -fi + mount -t glusterfs -o "${arr[3]}" "${arr[0]}" "${arr[1]}" + #wait for few seconds + sleep 10 + + #recheck mount got succeed + if mountpoint -q "${arr[1]}" + then + echo "${arr[1]} has been mounted" + continue + else + echo "${arr[1]} failed to mount" + exitStatus=1 + fi + fi +done <<< "$(sed '/^#/ d' </etc/fstab | grep 'glusterfs')" +exit $exitStatus diff --git a/extras/ocf/volume.in b/extras/ocf/volume.in index de05373b9e3..76cc649e55f 100755 --- a/extras/ocf/volume.in +++ b/extras/ocf/volume.in @@ -6,6 +6,7 @@ # HA resource # # Authors: Florian Haas (hastexo Professional Services GmbH) +# Jiri Lunacek (Hosting90 Systems s.r.o.) # # License: GNU General Public License (GPL) @@ -54,6 +55,14 @@ must have clone ordering enabled. <shortdesc lang="en">gluster executable</shortdesc> <content type="string" default="$OCF_RESKEY_binary_default"/> </parameter> + <parameter name="peer_map"> + <longdesc lang="en"> + Mapping of hostname - peer name in the gluster cluster + in format hostname1:peername1,hostname2:peername2,... + </longdesc> + <shortdesc lang="en">gluster peer map</shortdesc> + <content type="string" default=""/> + </parameter> </parameters> <actions> <action name="start" timeout="20" /> @@ -68,9 +77,13 @@ EOF } +if [ -n "${OCF_RESKEY_peer_map}" ]; then + SHORTHOSTNAME=`echo "${OCF_RESKEY_peer_map}" | egrep -o "$SHORTHOSTNAME\:[^,]+" | awk -F: '{print $2}'` +fi + volume_getdir() { local voldir - voldir="@sysconfdir@/glusterd/vols/${OCF_RESKEY_volname}" + voldir="@GLUSTERD_WORKDIR@/vols/${OCF_RESKEY_volname}" [ -d ${voldir} ] || return 1 @@ -108,6 +121,10 @@ volume_getpids() { volpid_dir=`volume_getpid_dir` bricks=`volume_getbricks` + + if [ -z "$bricks" ]; then + return 1 + fi for brick in ${bricks}; do pidfile="${volpid_dir}/${SHORTHOSTNAME}${brick}.pid" @@ -214,6 +231,11 @@ volume_validate_all() { # Test for required binaries check_binary $OCF_RESKEY_binary + + if [ -z "$SHORTHOSTNAME" ]; then + ocf_log err 'Unable to get host in node map' + return $OCF_ERR_CONFIGURED + fi return $OCF_SUCCESS } diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler index 65d445864aa..aaafd088648 100755 --- a/extras/profiler/glusterfs-profiler +++ b/extras/profiler/glusterfs-profiler @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> # This file is part of GlusterFS. @@ -291,7 +291,7 @@ class Texttable: s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()], horiz) # build the line - l = string.join([horiz*n for n in self._width], s) + l = s.join([horiz*n for n in self._width]) # add border if needed if self._has_border(): l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz, diff --git a/extras/prot_filter.py b/extras/prot_filter.py deleted file mode 100755 index 7dccacf155e..00000000000 --- a/extras/prot_filter.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python - -""" - Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com> - This file is part of GlusterFS. - - This file is licensed to you under your choice of the GNU Lesser - General Public License, version 3 or any later version (LGPLv3 or - later), or the GNU General Public License, version 2 (GPLv2), in all - cases as published by the Free Software Foundation. -""" - -""" - INSTRUCTIONS - Put this in /usr/lib64/glusterfs/$version/filter to have it run automatically, - or else you'll have to run it by hand every time you change the volume - configuration. Give it a list of volume names on which to enable the - protection functionality; it will deliberately ignore client volfiles for - other volumes, and all server volfiles. It *will* include internal client - volfiles such as those used for NFS or rebalance/self-heal; this is a - deliberate choice so that it will catch deletions from those sources as well. -""" - -volume_list = [ "jdtest" ] - -import copy -import string -import sys -import types - -class Translator: - def __init__ (self, name): - self.name = name - self.xl_type = "" - self.opts = {} - self.subvols = [] - self.dumped = False - def __repr__ (self): - return "<Translator %s>" % self.name - -def load (path): - # If it's a string, open it; otherwise, assume it's already a - # file-like object (most notably from urllib*). - if type(path) in types.StringTypes: - fp = file(path,"r") - else: - fp = path - all_xlators = {} - xlator = None - last_xlator = None - while True: - text = fp.readline() - if text == "": - break - text = text.split() - if not len(text): - continue - if text[0] == "volume": - if xlator: - raise RuntimeError, "nested volume definition" - xlator = Translator(text[1]) - continue - if not xlator: - raise RuntimeError, "text outside volume definition" - if text[0] == "type": - xlator.xl_type = text[1] - continue - if text[0] == "option": - xlator.opts[text[1]] = string.join(text[2:]) - continue - if text[0] == "subvolumes": - for sv in text[1:]: - xlator.subvols.append(all_xlators[sv]) - continue - if text[0] == "end-volume": - all_xlators[xlator.name] = xlator - last_xlator = xlator - xlator = None - continue - raise RuntimeError, "unrecognized keyword %s" % text[0] - if xlator: - raise RuntimeError, "unclosed volume definition" - return all_xlators, last_xlator - -def generate (graph, last, stream=sys.stdout): - for sv in last.subvols: - if not sv.dumped: - generate(graph,sv,stream) - print >> stream, "" - sv.dumped = True - print >> stream, "volume %s" % last.name - print >> stream, " type %s" % last.xl_type - for k, v in last.opts.iteritems(): - print >> stream, " option %s %s" % (k, v) - if last.subvols: - print >> stream, " subvolumes %s" % string.join( - [ sv.name for sv in last.subvols ]) - print >> stream, "end-volume" - -def push_filter (graph, old_xl, filt_type, opts={}): - new_type = "-" + filt_type.split("/")[1] - old_type = "-" + old_xl.xl_type.split("/")[1] - pos = old_xl.name.find(old_type) - if pos >= 0: - new_name = old_xl.name - old_name = new_name[:pos] + new_type + new_name[len(old_type)+pos:] - else: - new_name = old_xl.name + old_type - old_name = old_xl.name + new_type - new_xl = Translator(new_name) - new_xl.xl_type = old_xl.xl_type - new_xl.opts = old_xl.opts - new_xl.subvols = old_xl.subvols - graph[new_xl.name] = new_xl - old_xl.name = old_name - old_xl.xl_type = filt_type - old_xl.opts = opts - old_xl.subvols = [new_xl] - graph[old_xl.name] = old_xl - -if __name__ == "__main__": - path = sys.argv[1] - # Alow an override for debugging. - for extra in sys.argv[2:]: - volume_list.append(extra) - graph, last = load(path) - for v in volume_list: - if graph.has_key(v): - break - else: - print "No configured volumes found - aborting." - sys.exit(0) - for v in graph.values(): - if v.xl_type == "cluster/distribute": - push_filter(graph,v,"features/prot_dht") - elif v.xl_type == "protocol/client": - push_filter(graph,v,"features/prot_client") - # We push debug/trace so that every fop gets a real frame, because DHT - # gets confused if STACK_WIND_TAIL causes certain fops to be invoked - # from anything other than a direct child. - for v in graph.values(): - if v.xl_type == "features/prot_client": - push_filter(graph,v,"debug/trace") - generate(graph,last,stream=open(path,"w")) diff --git a/extras/python/Makefile.am b/extras/python/Makefile.am new file mode 100644 index 00000000000..7d81fa0319b --- /dev/null +++ b/extras/python/Makefile.am @@ -0,0 +1,7 @@ +if HAVE_PYTHON +# Install __init__.py into the Python site-packages area +pypkgdir = @BUILD_PYTHON_SITE_PACKAGES@/gluster +pypkg_PYTHON = __init__.py +endif + +EXTRA_DIST = __init__.py diff --git a/extras/python/__init__.py b/extras/python/__init__.py new file mode 100644 index 00000000000..3ad9513f40e --- /dev/null +++ b/extras/python/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/extras/quota/log_accounting.sh b/extras/quota/log_accounting.sh index 9c2381f6a95..e2dd87b84d7 100755 --- a/extras/quota/log_accounting.sh +++ b/extras/quota/log_accounting.sh @@ -1,4 +1,4 @@ -#!/usr/bin/bash +#!/bin/bash # The script does an accounting of all directories using command 'du' and # using gluster. We can then compare the two to identify accounting mismatch # THere can be minor mismatch because gluster only accounts for the size of diff --git a/extras/quota/quota_fsck.py b/extras/quota/quota_fsck.py new file mode 100755 index 00000000000..e62f7fc52a3 --- /dev/null +++ b/extras/quota/quota_fsck.py @@ -0,0 +1,377 @@ +#!/usr/bin/python3 +# The following script enables, Detecting, Reporting and Fixing +# anomalies in quota accounting. Run this script with -h option +# for further details. + +''' + Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +''' +from __future__ import print_function +import os, sys, re +from stat import * +import subprocess +import argparse +import xattr + +aggr_size = {} +verbose_mode = False +mnt_path = None +brick_path = None +obj_fix_count = 0 +file_count = 0 +dir_count = 0 + +#CONSTANTS +KB = 1024 +MB = 1048576 +GB = 1048576 * 1024 +TB = 1048576 * 1048576 + +QUOTA_VERBOSE = 0 +QUOTA_META_ABSENT = 1 +QUOTA_SIZE_MISMATCH = 2 + +IS_DIRTY ='0x3100' +IS_CLEAN ='0x3000' + + +epilog_msg=''' + The script attempts to find any gluster accounting issues in the + filesystem at the given subtree. The script crawls the given + subdirectory tree doing a stat for all files and compares the + size reported by gluster quota with the size reported by stat + calls. Any mismatch is reported. In addition integrity of marker + xattrs are verified. + ''' + +def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None): + if log_type == QUOTA_VERBOSE: + print('%-24s %-60s\nxattr_values: %s\n%s\n' % ("Verbose", path, xattr_dict, stbuf)) + elif log_type == QUOTA_META_ABSENT: + print('%-24s %-60s\n%s\n' % ("Quota-Meta Absent", path, xattr_dict)) + elif log_type == QUOTA_SIZE_MISMATCH: + print("mismatch") + if dir_size is not None: + print('%24s %60s %12s %12s' % ("Size Mismatch", path, + xattr_dict, dir_size)) + else: + print('%-24s %-60s %-12s %-12s' % ("Size Mismatch", path, xattr_dict, + stbuf.st_size)) + +def size_differs_lot(s1, s2): + ''' + There could be minor accounting differences between the stat based + accounting and gluster accounting. To avoid these from throwing lot + of false positives in our logs. using a threshold of 1M for now. + TODO: For a deeply nested directory, at higher levels in hierarchy + differences may not be significant, hence this check needs to be improved. + ''' + if abs(s1-s2) > 0: + return True + else: + return False + +def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size): + ''' + Hard links are messy.. we have to account them for their parent + directory. But, stop accounting at the most common ancestor. + Eg: + say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3 + + suppose we encounter the hard links h1 first , then h2 and then h3. + while accounting for h1, we account the size until root(d2->d1->/) + while accounting for h2, we need to account only till d3. (as d1 + and / are accounted for this inode). + while accounting for h3 we should not account at all.. as all + its ancestors are already accounted for same inode. + + curr_dict : dict of hardlinks that were seen and + accounted by the current iteration. + accounted_dict : dict of hardlinks that has already been + accounted for. + + size : size of the object as accounted by the + curr_iteration. + + Return vale: + curr_size : size reduced by hardlink sizes for those + hardlinks that has already been accounted + in current subtree. + Also delete the duplicate link from curr_dict. + ''' + + dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys()) + for link in dual_accounted_links: + curr_size = curr_size - curr_dict[link] + del curr_dict[link] + return curr_size + + +def fix_xattr(file_name, mark_dirty): + global obj_fix_count + global mnt_path + + if mnt_path is None: + return + if mark_dirty: + print("MARKING DIRTY: " + file_name) + out = subprocess.check_output (["/usr/bin/setfattr", "-n", + "trusted.glusterfs.quota.dirty", + "-v", IS_DIRTY, file_name]) + rel_path = os.path.relpath(file_name, brick_path) + print("stat on " + mnt_path + "/" + rel_path) + stbuf = os.lstat(mnt_path + "/" + rel_path) + + obj_fix_count += 1 + +def get_quota_xattr_brick(dpath): + out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference", + "-d", "-m.", "-e", "hex", dpath]) + pairs = out.splitlines() + + ''' + Sample output to be parsed: + [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/ + # file: export/b1/B0/d14/d13/ + security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000 + trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8 + trusted.glusterfs.dht=0x000000010000000000000000ffffffff + trusted.glusterfs.dht.mds=0x00000000 + trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e + trusted.glusterfs.quota.dirty=0x3000 + trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e + ''' + + ''' + xattr_dict dictionary holds quota related xattrs + eg: + ''' + + xattr_dict = {} + xattr_dict['parents'] = {} + + for xattr in pairs[1:]: + xattr = xattr.decode("utf-8") + xattr_key = xattr.split("=")[0] + if xattr_key == "": + # skip any empty lines + continue + elif not re.search("quota", xattr_key): + # skip all non quota xattr. + continue + + xattr_value = xattr.split("=")[1] + if re.search("contri", xattr_key): + + xattr_version = xattr_key.split(".")[5] + if 'version' not in xattr_dict: + xattr_dict['version'] = xattr_version + else: + if xattr_version != xattr_dict['version']: + print("Multiple xattr version found") + + + cur_parent = xattr_key.split(".")[3] + if cur_parent not in xattr_dict['parents']: + xattr_dict['parents'][cur_parent] = {} + + contri_dict = xattr_dict['parents'][cur_parent] + if len(xattr_value) == 34: + # 34 bytes implies file contri xattr + # contri format =0x< 16bytes file size><16bytes file count> + # size is obtained in iatt, file count = 1, dir count=0 + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = 0 + else: + # This is a directory contri. + contri_dict['contri_size'] = int(xattr_value[2:18], 16) + contri_dict['contri_file_count'] = int(xattr_value[18:34], 16) + contri_dict['contri_dir_count'] = int(xattr_value[34:], 16) + + elif re.search("size", xattr_key): + xattr_dict['size'] = int(xattr_value[2:18], 16) + xattr_dict['file_count'] = int(xattr_value[18:34], 16) + xattr_dict['dir_count'] = int(xattr_value[34:], 16) + elif re.search("dirty", xattr_key): + if xattr_value == IS_CLEAN: + xattr_dict['dirty'] = False + elif xattr_value == IS_DIRTY: + xattr_dict['dirty'] = True + elif re.search("limit_objects", xattr_key): + xattr_dict['limit_objects'] = int(xattr_value[2:18], 16) + elif re.search("limit_set", xattr_key): + xattr_dict['limit_set'] = int(xattr_value[2:18], 16) + + return xattr_dict + +def verify_file_xattr(path, stbuf = None): + + global file_count + file_count += 1 + + if stbuf is None: + stbuf = os.lstat(path) + + xattr_dict = get_quota_xattr_brick(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'contri_size' not in contri_dict or \ + 'contri_file_count' not in contri_dict or \ + 'contri_dir_count' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf) + fix_xattr(path, False) + return + + if verbose_mode is True: + print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf) + + +def verify_dir_xattr(path, dir_size): + + global dir_count + dir_count += 1 + xattr_dict = get_quota_xattr_brick(path) + + stbuf = os.lstat(path) + + for parent in xattr_dict['parents']: + contri_dict = xattr_dict['parents'][parent] + + if 'size' not in xattr_dict or 'contri_size' not in contri_dict: + print_msg(QUOTA_META_ABSENT, path) + fix_xattr(path, True) + return + elif size_differs_lot(dir_size, xattr_dict['size']) or \ + size_differs_lot(contri_dict['contri_size'], xattr_dict['size']): + print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size) + fix_xattr(path, True) + return + + if verbose_mode is True: + print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size) + + +def walktree(t_dir, hard_link_dict): + '''recursively descend the directory tree rooted at dir, + aggregating the size + t_dir : directory to walk over. + hard_link_dict : dict of inodes with multiple hard_links under t_dir + ''' + global aggr_size + aggr_size[t_dir] = 0 + + for entry in os.listdir(t_dir): + pathname = os.path.join(t_dir, entry) + stbuf = os.lstat(pathname) + if S_ISDIR(stbuf.st_mode): + # It's a directory, recurse into it + if entry == '.glusterfs': + print("skipping " + pathname) + continue + descendent_hardlinks = {} + subtree_size = walktree(pathname, descendent_hardlinks) + + subtree_size = fix_hardlink_accounting(descendent_hardlinks, + hard_link_dict, + subtree_size) + + aggr_size[t_dir] = aggr_size[t_dir] + subtree_size + + elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode): + # Even a symbolic link file may have multiple hardlinks. + + file_size = stbuf.st_size + if stbuf.st_nlink > 2: + # send a single element dict to check if file is accounted. + file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size}, + hard_link_dict, + stbuf.st_size) + + if file_size == 0: + print_msg("HARD_LINK (skipped)", pathname, "", + stbuf) + else: + print_msg("HARD_LINK (accounted)", pathname, "", + stbuf) + hard_link_dict[stbuf.st_ino] = stbuf.st_size + + if t_dir in aggr_size: + aggr_size[t_dir] = aggr_size[t_dir] + file_size + else: + aggr_size[t_dir] = file_size + verify_file_xattr(pathname, stbuf) + + else: + # Unknown file type, print a message + print('Skipping %s, due to file mode' % (pathname)) + + if t_dir not in aggr_size: + aggr_size[t_dir] = 0 + + verify_dir_xattr(t_dir, aggr_size[t_dir]) + # du also accounts for t_directory sizes + # aggr_size[t_dir] += 4096 + + #cleanup + ret = aggr_size[t_dir] + del aggr_size[t_dir] + return ret + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg) + parser.add_argument('brick_path', nargs=1, + help='The brick path (or any descendent sub-directory of brick path)', + ) + parser.add_argument('--full-logs', dest='verbose', action='store_true', + help=''' + log all the xattr values and stat values reported + for analysis. [CAUTION: This can give lot of output + depending on FS depth. So one has to make sure enough + disk space exists if redirecting to file] + ''' + ) + parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store', + help=''' + fix accounting issues where the xattr values disagree + with stat sizes reported by gluster. A mount is also + required for this option to be used. + [CAUTION: This will directly modify backend xattr] + ''' + ) + parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store', + help=''' + limit the crawling and accounting verification/correction + to a specific subdirectory. + ''' + ) + + args = parser.parse_args() + verbose_mode = args.verbose + brick_path = args.brick_path[0] + sub_dir = args.sub_dir + mnt_path = args.mnt + hard_link_dict = {} + if sub_dir is not None: + walktree(os.path.join(brick_path, sub_dir), hard_link_dict) + else: + walktree(brick_path, hard_link_dict) + + print("Files verified : " + str(file_count)) + print("Directories verified : " + str(dir_count)) + if mnt_path is not None: + print("Objects Fixed : " + str(obj_fix_count)) diff --git a/extras/quota/xattr_analysis.py b/extras/quota/xattr_analysis.py index d3d1a74170b..7bd7d96374c 100755 --- a/extras/quota/xattr_analysis.py +++ b/extras/quota/xattr_analysis.py @@ -1,12 +1,13 @@ -#!/usr/bin/python -# Below script has to purpose +#!/usr/bin/python3 +# Below script has two purposes # 1. Display xattr of entire FS tree in a human readable form # 2. Display all the directory where contri and size mismatch. # (If there are any directory with contri and size mismatch that are not dirty -# then that highlights a propogation issue) +# then that highlights a propagation issue) # The script takes only one input LOG _FILE generated from the command, # find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr +from __future__ import print_function import re import subprocess import sys @@ -14,7 +15,7 @@ from hurry.filesize import size if len(sys.argv) < 2: sys.exit('Usage: %s log_gluster_xattr \n' - 'to genereate log_gluster_xattr use: \n' + 'to generate log_gluster_xattr use: \n' 'find <brick_path> | xargs getfattr -d -m. -e hex > log_gluster_xattr' % sys.argv[0]) LOG_FILE=sys.argv[1] @@ -27,17 +28,17 @@ def get_quota_xattr_brick(): mismatch_size = [('====contri_size===', '====size====')] for xattr in pairs: k = xattr.split("=")[0] - if re.search("# file:",k): - print xdict + if re.search("# file:", k): + print(xdict) filename=k - print "=====" + filename + "=======" + print("=====" + filename + "=======") xdict = {} elif k is "": pass else: - print xattr + print(xattr) v = xattr.split("=")[1] - if re.search("contri",k): + if re.search("contri", k): if len(v) == 34: # for files size is obtained in iatt, file count should be 1, dir count=0 xdict['contri_file_count'] = int(v[18:34], 16) @@ -46,25 +47,25 @@ def get_quota_xattr_brick(): xdict['contri_size'] = size(int(v[2:18], 16)) xdict['contri_file_count'] = int(v[18:34], 16) xdict['contri_dir_count'] = int(v[34:], 16) - elif re.search("size",k): + elif re.search("size", k): xdict['size'] = size(int(v[2:18], 16)) xdict['file_count'] = int(v[18:34], 16) xdict['dir_count'] = int(v[34:], 16) - elif re.search("dirty",k): + elif re.search("dirty", k): if v == '0x3000': xdict['dirty'] = False elif v == '0x3100': xdict['dirty'] = True - elif re.search("limit_objects",k): + elif re.search("limit_objects", k): xdict['limit_objects'] = int(v[2:18], 16) - elif re.search("limit_set",k): + elif re.search("limit_set", k): xdict['limit_set'] = size(int(v[2:18], 16)) if 'size' in xdict and 'contri_size' in xdict and xdict['size'] != xdict['contri_size']: mismatch_size.append((xdict['contri_size'], xdict['size'], filename)) for values in mismatch_size: - print values + print(values) if __name__ == '__main__': diff --git a/extras/rebalance.py b/extras/rebalance.py index 9579e5616ad..37c68ebbb42 100755 --- a/extras/rebalance.py +++ b/extras/rebalance.py @@ -1,4 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 + +from __future__ import print_function import atexit import copy @@ -38,20 +40,20 @@ class Brick: def get_bricks (host, vol): t = pipes.Template() - t.prepend("gluster --remote-host=%s system getspec %s"%(host,vol),".-") - return t.open(None,"r") + t.prepend("gluster --remote-host=%s system getspec %s"%(host, vol), ".-") + return t.open(None, "r") def generate_stanza (vf, all_xlators, cur_subvol): sv_list = [] for sv in cur_subvol.subvols: - generate_stanza(vf,all_xlators,sv) + generate_stanza(vf, all_xlators, sv) sv_list.append(sv.name) - vf.write("volume %s\n"%cur_subvol.name) - vf.write(" type %s\n"%cur_subvol.type) - for kvpair in cur_subvol.opts.iteritems(): - vf.write(" option %s %s\n"%kvpair) + vf.write("volume %s\n" % cur_subvol.name) + vf.write(" type %s\n" % cur_subvol.type) + for kvpair in cur_subvol.opts.items(): + vf.write(" option %s %s\n" % kvpair) if sv_list: - vf.write(" subvolumes %s\n"%string.join(sv_list)) + vf.write(" subvolumes %s\n" % ''.join(sv_list)) vf.write("end-volume\n\n") @@ -59,14 +61,14 @@ def mount_brick (localpath, all_xlators, dht_subvol): # Generate a volfile. vf_name = localpath + ".vol" - vf = open(vf_name,"w") - generate_stanza(vf,all_xlators,dht_subvol) + vf = open(vf_name, "w") + generate_stanza(vf, all_xlators, dht_subvol) vf.flush() vf.close() # Create a brick directory and mount the brick there. os.mkdir(localpath) - subprocess.call(["glusterfs","-f",vf_name,localpath]) + subprocess.call(["glusterfs", "-f", vf_name, localpath]) # We use the command-line tools because there's no getxattr support in the # Python standard library (which is ridiculous IMO). Adding the xattr package @@ -80,16 +82,16 @@ def mount_brick (localpath, all_xlators, dht_subvol): def get_range (brick): t = pipes.Template() cmd = "getfattr -e hex -n trusted.glusterfs.dht %s 2> /dev/null" - t.prepend(cmd%brick,".-") - t.append("grep ^trusted.glusterfs.dht=","--") - f = t.open(None,"r") + t.prepend(cmd%brick, ".-") + t.append("grep ^trusted.glusterfs.dht=", "--") + f = t.open(None, "r") try: value = f.readline().rstrip().split('=')[1][2:] except: - print "could not get layout for %s (might be OK)" % brick + print("could not get layout for %s (might be OK)" % brick) return None - v_start = int("0x"+value[16:24],16) - v_end = int("0x"+value[24:32],16) + v_start = int("0x"+value[16:24], 16) + v_end = int("0x"+value[24:32], 16) return (v_start, v_end) def calc_sizes (bricks, total): @@ -126,7 +128,7 @@ def normalize (in_bricks): curr_hash = b.r_end + 1 break else: - print "gap found at 0x%08x" % curr_hash + print("gap found at 0x%08x" % curr_hash) sys.exit(1) return out_bricks + in_bricks, used @@ -154,8 +156,8 @@ def get_score (bricks): if __name__ == "__main__": - my_usage = "%prog [options] server volume [directory]" - parser = optparse.OptionParser(usage=my_usage) + my_usage = "%prog [options] server volume [directory]" + parser = optparse.OptionParser(usage=my_usage) parser.add_option("-f", "--free-space", dest="free_space", default=False, action="store_true", help="use free space instead of total space") @@ -165,7 +167,7 @@ if __name__ == "__main__": parser.add_option("-v", "--verbose", dest="verbose", default=False, action="store_true", help="verbose output") - options, args = parser.parse_args() + options, args = parser.parse_args() if len(args) == 3: fix_dir = args[2] @@ -183,9 +185,9 @@ if __name__ == "__main__": def cleanup_workdir (): os.chdir(orig_dir) if options.verbose: - print "Cleaning up %s" % work_dir + print("Cleaning up %s" % work_dir) for b in bricks: - subprocess.call(["umount",b.path]) + subprocess.call(["umount", b.path]) shutil.rmtree(work_dir) if not options.leave_mounted: atexit.register(cleanup_workdir) @@ -193,29 +195,29 @@ if __name__ == "__main__": # Mount each brick individually, so we can issue brick-specific calls. if options.verbose: - print "Mounting subvolumes..." + print("Mounting subvolumes...") index = 0 - volfile_pipe = get_bricks(hostname,volname) + volfile_pipe = get_bricks(hostname, volname) all_xlators, last_xlator = volfilter.load(volfile_pipe) for dht_vol in all_xlators.itervalues(): if dht_vol.type == "cluster/distribute": break else: - print "no DHT volume found" + print("no DHT volume found") sys.exit(1) for sv in dht_vol.subvols: #print "found subvol %s" % sv.name lpath = "%s/brick%s" % (work_dir, index) index += 1 - mount_brick(lpath,all_xlators,sv) - bricks.append(Brick(lpath,sv.name)) + mount_brick(lpath, all_xlators, sv) + bricks.append(Brick(lpath, sv.name)) if index == 0: - print "no bricks" + print("no bricks") sys.exit(1) # Collect all of the sizes. if options.verbose: - print "Collecting information..." + print("Collecting information...") total = 0 for b in bricks: info = os.statvfs(b.path) @@ -237,7 +239,7 @@ if __name__ == "__main__": else: size = info[2] / blocksper100mb if size <= 0: - print "brick %s has invalid size %d" % (b.path, size) + print("brick %s has invalid size %d" % (b.path, size)) sys.exit(1) b.set_size(size) total += size @@ -248,13 +250,13 @@ if __name__ == "__main__": if hash_range is not None: rs, re = hash_range if rs > re: - print "%s has backwards hash range" % b.path + print("%s has backwards hash range" % b.path) sys.exit(1) - b.set_range(hash_range[0],hash_range[1]) + b.set_range(hash_range[0], hash_range[1]) if options.verbose: - print "Calculating new layouts..." - calc_sizes(bricks,total) + print("Calculating new layouts...") + calc_sizes(bricks, total) bricks, used = normalize(bricks) # We can't afford O(n!) here, but O(n^2) should be OK and the result @@ -262,10 +264,10 @@ if __name__ == "__main__": while used < len(bricks): best_place = used best_score = get_score(bricks) - for i in xrange(used): + for i in range(used): new_bricks = bricks[:] del new_bricks[used] - new_bricks.insert(i,bricks[used]) + new_bricks.insert(i, bricks[used]) new_score = get_score(new_bricks) if new_score > best_score: best_place = i @@ -273,7 +275,7 @@ if __name__ == "__main__": if best_place != used: nb = bricks[used] del bricks[used] - bricks.insert(best_place,nb) + bricks.insert(best_place, nb) used += 1 # Finalize whatever we decided on. @@ -283,25 +285,25 @@ if __name__ == "__main__": curr_hash += b.good_size b.r_end = curr_hash - 1 - print "Here are the xattr values for your size-weighted layout:" + print("Here are the xattr values for your size-weighted layout:") for b in bricks: - print " %s: 0x0000000200000000%08x%08x" % ( - b.sv_name, b.r_start, b.r_end) + print(" %s: 0x0000000200000000%08x%08x" % ( + b.sv_name, b.r_start, b.r_end)) if fix_dir: if options.verbose: - print "Fixing layout for %s" % fix_dir + print("Fixing layout for %s" % fix_dir) for b in bricks: value = "0x0000000200000000%08x%08x" % ( b.r_start, b.r_end) path = "%s/%s" % (b.path, fix_dir) cmd = "setfattr -n trusted.glusterfs.dht -v %s %s" % ( value, path) - print cmd + print(cmd) if options.leave_mounted: - print "The following subvolumes are still mounted:" + print("The following subvolumes are still mounted:") for b in bricks: - print "%s on %s" % (b.sv_name, b.path) - print "Don't forget to clean up when you're done." + print("%s on %s" % (b.sv_name, b.path)) + print("Don't forget to clean up when you're done.") diff --git a/extras/snap_scheduler/Makefile.am b/extras/snap_scheduler/Makefile.am index ffc157935a3..782f139016f 100644 --- a/extras/snap_scheduler/Makefile.am +++ b/extras/snap_scheduler/Makefile.am @@ -1,6 +1,8 @@ snap_schedulerdir = $(sbindir)/ +if WITH_SERVER snap_scheduler_SCRIPTS = gcron.py snap_scheduler.py conf.py +endif EXTRA_DIST = gcron.py snap_scheduler.py conf.py diff --git a/extras/snap_scheduler/gcron.py b/extras/snap_scheduler/gcron.py index d72057861ff..0e4df77d481 100755 --- a/extras/snap_scheduler/gcron.py +++ b/extras/snap_scheduler/gcron.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # # Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> # This file is part of GlusterFS. @@ -19,10 +19,10 @@ import logging.handlers import fcntl -GCRON_TASKS = "/var/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" +GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag" -LOCK_FILE_DIR = "/var/run/gluster/shared_storage/snaps/lock_files/" +LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/" log = logging.getLogger("gcron-logger") start_time = 0.0 @@ -38,7 +38,8 @@ def initLogger(script_name): sh.setFormatter(formatter) process = subprocess.Popen(["gluster", "--print-logdir"], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() if process.returncode == 0: logfile = os.path.join(out.strip(), script_name[:-3]+".log") @@ -88,7 +89,7 @@ def takeSnap(volname="", snapname=""): def doJob(name, lockFile, jobFunc, volname): success = True try: - f = os.open(lockFile, os.O_RDWR | os.O_NONBLOCK) + f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK) try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) mtime = os.path.getmtime(lockFile) @@ -105,11 +106,11 @@ def doJob(name, lockFile, jobFunc, volname): else: log.info("Job %s has been processed already", name) fcntl.flock(f, fcntl.LOCK_UN) - except IOError as (errno, strerror): + except (OSError, IOError): log.info("Job %s is being processed by another agent", name) os.close(f) - except IOError as (errno, strerror): - log.debug("Failed to open lock file %s : %s", lockFile, strerror) + except (OSError, IOError) as e: + log.debug("Failed to open lock file %s : %s", lockFile, e) log.error("Failed to process job %s", name) success = False @@ -122,19 +123,20 @@ def main(): global start_time if sys.argv[1] == "--update": if not os.path.exists(GCRON_TASKS): - # Create a flag in /var/run/gluster which indicates that this nodes - # doesn't have access to GCRON_TASKS right now, so that + # Create a flag in /var/run/gluster which indicates that this + # node doesn't have access to GCRON_TASKS right now, so that # when the mount is available and GCRON_TASKS is available # the flag will tell this routine to reload GCRON_CROND_TASK try: - f = os.open(GCRON_RELOAD_FLAG, os.O_CREAT | os.O_NONBLOCK, 0644) + f = os.open(GCRON_RELOAD_FLAG, + os.O_CREAT | os.O_NONBLOCK, 0o644) os.close(f) - except OSError as (errno, strerror): + except OSError as e: if errno != EEXIST: log.error("Failed to create %s : %s", - GCRON_RELOAD_FLAG, strerror) + GCRON_RELOAD_FLAG, e) output("Failed to create %s. Error: %s" - % (GCRON_RELOAD_FLAG, strerror)) + % (GCRON_RELOAD_FLAG, e)) return if not os.path.exists(GCRON_CROND_TASK): @@ -153,9 +155,9 @@ def main(): if process.returncode != 0: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, err) - except (IOError, OSError) as (errno, strerror): + except (IOError, OSError) as e: log.error("Failed to touch %s. Error: %s.", - GCRON_CROND_TASK, strerror) + GCRON_CROND_TASK, e) return if os.lstat(GCRON_TASKS).st_mtime > \ os.lstat(GCRON_CROND_TASK).st_mtime: @@ -167,9 +169,9 @@ def main(): if process.returncode != 0: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, err) - except IOError as (errno, strerror): + except IOError as e: log.error("Failed to touch %s. Error: %s.", - GCRON_CROND_TASK, strerror) + GCRON_CROND_TASK, e) return volname = sys.argv[1] diff --git a/extras/snap_scheduler/snap_scheduler.py b/extras/snap_scheduler/snap_scheduler.py index 61d1c514a9a..e8fcc449a9b 100755 --- a/extras/snap_scheduler/snap_scheduler.py +++ b/extras/snap_scheduler/snap_scheduler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # # Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> # This file is part of GlusterFS. @@ -67,7 +67,7 @@ except ImportError: SCRIPT_NAME = "snap_scheduler" scheduler_enabled = False log = logging.getLogger(SCRIPT_NAME) -SHARED_STORAGE_DIR="/var/run/gluster/shared_storage" +SHARED_STORAGE_DIR="/run/gluster/shared_storage" GCRON_DISABLED = SHARED_STORAGE_DIR+"/snaps/gcron_disabled" GCRON_ENABLED = SHARED_STORAGE_DIR+"/snaps/gcron_enabled" GCRON_TASKS = SHARED_STORAGE_DIR+"/snaps/glusterfs_snap_cron_tasks" @@ -149,7 +149,7 @@ def initLogger(): sh.setFormatter(formatter) process = subprocess.Popen(["gluster", "--print-logdir"], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, universal_newlines=True) logfile = os.path.join(process.stdout.read()[:-1], SCRIPT_NAME + ".log") fh = logging.FileHandler(logfile) @@ -207,11 +207,11 @@ def enable_scheduler(): os.remove(GCRON_TASKS) try: f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, - 0644) + 0o644) os.close(f) - except OSError as (errno, strerror): + except OSError as e: log.error("Failed to open %s. Error: %s.", - GCRON_ENABLED, strerror) + GCRON_ENABLED, e) ret = INTERNAL_ERROR return ret os.symlink(GCRON_ENABLED, GCRON_TASKS) @@ -219,8 +219,9 @@ def enable_scheduler(): log.info("Snapshot scheduling is enabled") output("Snapshot scheduling is enabled") ret = 0 - except OSError as (errno, strerror): - print_str = "Failed to enable snapshot scheduling. Error: "+strerror + except OSError as e: + print_str = ("Failed to enable snapshot scheduling." + "Error: {{}}" + e) log.error(print_str) output(print_str) ret = INTERNAL_ERROR @@ -262,14 +263,15 @@ def disable_scheduler(): os.remove(GCRON_DISABLED) if os.path.lexists(GCRON_TASKS): os.remove(GCRON_TASKS) - f = os.open(GCRON_DISABLED, os.O_CREAT, 0644) + f = os.open(GCRON_DISABLED, os.O_CREAT, 0o644) os.close(f) os.symlink(GCRON_DISABLED, GCRON_TASKS) log.info("Snapshot scheduling is disabled") output("Snapshot scheduling is disabled") ret = 0 - except OSError as (errno, strerror): - print_str = "Failed to disable snapshot scheduling. Error: "+strerror + except OSError as e: + print_str = ("Failed to disable snapshot scheduling. Error: " + + e) log.error(print_str) output(print_str) ret = INTERNAL_ERROR @@ -308,8 +310,8 @@ def load_tasks_from_file(): tasks[jobname] = schedule+":"+volname f.close() ret = 0 - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_ENABLED, e) ret = INTERNAL_ERROR return ret @@ -322,8 +324,8 @@ def get_current_scheduler(): current_scheduler = f.readline().rstrip('\n') f.close() ret = 0 - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", CURRENT_SCHEDULER, e) ret = INTERNAL_ERROR return ret @@ -363,7 +365,7 @@ def list_schedules(): def write_tasks_to_file(): try: - with open(TMP_FILE, "w", 0644) as f: + with open(TMP_FILE, "w", 0o644) as f: # If tasks is empty, just create an empty tmp file if len(tasks) != 0: for key in sorted(tasks): @@ -376,8 +378,8 @@ def write_tasks_to_file(): f.flush() os.fsync(f.fileno()) f.close() - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) ret = INTERNAL_ERROR return ret @@ -388,13 +390,13 @@ def write_tasks_to_file(): def update_current_scheduler(data): try: - with open(TMP_FILE, "w", 0644) as f: + with open(TMP_FILE, "w", 0o644) as f: f.write("%s" % data) f.flush() os.fsync(f.fileno()) f.close() - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) ret = INTERNAL_ERROR return ret @@ -457,11 +459,11 @@ def add_schedules(jobname, schedule, volname): job_lockfile = LOCK_FILE_DIR + jobname try: f = os.open(job_lockfile, os.O_CREAT | os.O_NONBLOCK, - 0644) + 0o644) os.close(f) - except OSError as (errno, strerror): + except OSError as e: log.error("Failed to open %s. Error: %s.", - job_lockfile, strerror) + job_lockfile, e) ret = INTERNAL_ERROR return ret log.info("Successfully added snapshot schedule %s" % @@ -489,9 +491,9 @@ def delete_schedules(jobname): job_lockfile = LOCK_FILE_DIR+jobname try: os.remove(job_lockfile) - except OSError as (errno, strerror): + except OSError as e: log.error("Failed to open %s. Error: %s.", - job_lockfile, strerror) + job_lockfile, e) ret = INTERNAL_ERROR return ret log.info("Successfully deleted snapshot schedule %s" @@ -575,8 +577,13 @@ def get_selinux_status(): getenforce_cli = ["getenforce"] log.debug("Running command '%s'", " ".join(getenforce_cli)) - p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + try: + p1 = subprocess.Popen(getenforce_cli, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as oserr: + log.error("Failed to run the command \"getenforce\". Error: %s" %\ + oserr) + return -1 output, err = p1.communicate() rv = p1.returncode @@ -638,15 +645,15 @@ def initialise_scheduler(): return ret try: - with open(TMP_FILE, "w+", 0644) as f: + with open(TMP_FILE, "w+", 0o644) as f: updater = ("* * * * * root PATH=$PATH:/usr/local/sbin:" "/usr/sbin gcron.py --update\n") f.write("%s\n" % updater) f.flush() os.fsync(f.fileno()) f.close() - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", TMP_FILE, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", TMP_FILE, e) ret = INIT_FAILED return ret @@ -654,10 +661,10 @@ def initialise_scheduler(): if not os.path.lexists(GCRON_TASKS): try: - f = open(GCRON_TASKS, "w", 0644) + f = open(GCRON_TASKS, "w", 0o644) f.close() - except IOError as (errno, strerror): - log.error("Failed to open %s. Error: %s.", GCRON_TASKS, strerror) + except IOError as e: + log.error("Failed to open %s. Error: %s.", GCRON_TASKS, e) ret = INIT_FAILED return ret @@ -725,7 +732,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, - status="Successfuly Disabled") + status="Successfully Disabled") else: gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, error=print_error(ret)) @@ -760,7 +767,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLED, - status="Successfuly Enabled") + status="Successfully Enabled") else: gf_event (EVENT_SNAPSHOT_SCHEDULER_ENABLE_FAILED, error=print_error(ret)) @@ -772,7 +779,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLED, - status="Successfuly Disabled") + status="Successfully Disabled") else: gf_event (EVENT_SNAPSHOT_SCHEDULER_DISABLE_FAILED, error=print_error(ret)) @@ -792,7 +799,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADDED, - status="Successfuly added job "+args.jobname) + status="Successfully added job "+args.jobname) else: gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_ADD_FAILED, status="Failed to add job "+args.jobname, @@ -808,7 +815,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETED, - status="Successfuly deleted job "+args.jobname) + status="Successfully deleted job "+args.jobname) else: gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_DELETE_FAILED, status="Failed to delete job "+args.jobname, @@ -824,7 +831,7 @@ def perform_operation(args): if ret == 0: subprocess.Popen(["touch", "-h", GCRON_TASKS]) gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDITED, - status="Successfuly edited job "+args.jobname) + status="Successfully edited job "+args.jobname) else: gf_event (EVENT_SNAPSHOT_SCHEDULER_SCHEDULE_EDIT_FAILED, status="Failed to edit job "+args.jobname, @@ -889,42 +896,42 @@ def main(argv): if not os.path.exists(SHARED_STORAGE_DIR+"/snaps/"): try: os.makedirs(SHARED_STORAGE_DIR+"/snaps/") - except OSError as (errno, strerror): + except OSError as e: if errno != EEXIST: - log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", strerror) + log.error("Failed to create %s : %s", SHARED_STORAGE_DIR+"/snaps/", e) output("Failed to create %s. Error: %s" - % (SHARED_STORAGE_DIR+"/snaps/", strerror)) + % (SHARED_STORAGE_DIR+"/snaps/", e)) return INTERNAL_ERROR if not os.path.exists(GCRON_ENABLED): - f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0644) + f = os.open(GCRON_ENABLED, os.O_CREAT | os.O_NONBLOCK, 0o644) os.close(f) if not os.path.exists(LOCK_FILE_DIR): try: os.makedirs(LOCK_FILE_DIR) - except OSError as (errno, strerror): + except OSError as e: if errno != EEXIST: - log.error("Failed to create %s : %s", LOCK_FILE_DIR, strerror) + log.error("Failed to create %s : %s", LOCK_FILE_DIR, e) output("Failed to create %s. Error: %s" - % (LOCK_FILE_DIR, strerror)) + % (LOCK_FILE_DIR, e)) return INTERNAL_ERROR try: - f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0644) + f = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK, 0o644) try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) ret = perform_operation(args) fcntl.flock(f, fcntl.LOCK_UN) - except IOError as (errno, strerror): + except IOError: log.info("%s is being processed by another agent.", LOCK_FILE) output("Another snap_scheduler command is running. " "Please try again after some time.") return ANOTHER_TRANSACTION_IN_PROGRESS os.close(f) - except OSError as (errno, strerror): - log.error("Failed to open %s : %s", LOCK_FILE, strerror) - output("Failed to open %s. Error: %s" % (LOCK_FILE, strerror)) + except OSError as e: + log.error("Failed to open %s : %s", LOCK_FILE, e) + output("Failed to open %s. Error: %s" % (LOCK_FILE, e)) return INTERNAL_ERROR return ret diff --git a/extras/statedumpparse.rb b/extras/statedumpparse.rb new file mode 100755 index 00000000000..1aff43377db --- /dev/null +++ b/extras/statedumpparse.rb @@ -0,0 +1,208 @@ +#!/usr/bin/env ruby + +require 'time' +require 'optparse' + +unless Array.instance_methods.include? :to_h + class Array + def to_h + h = {} + each { |k,v| h[k]=v } + h + end + end +end + +# statedump.c:gf_proc_dump_mempool_info uses a five-dash record separator, +# client.c:client_fd_lk_ctx_dump uses a six-dash record separator. +ARRSEP = /^(-{5,6}=-{5,6})?$/ +HEAD = /^\[(.*)\]$/ +INPUT_FORMATS = %w[statedump json] + +format = 'json' +input_format = 'statedump' +tz = '+0000' +memstat_select,memstat_reject = //,/\Z./ +OptionParser.new do |op| + op.banner << " [<] <STATEDUMP>" + op.on("-f", "--format=F", "json/yaml/memstat(-[plain|human|json])") { |s| format = s } + op.on("--input-format=F", INPUT_FORMATS.join(?/)) { |s| input_format = s } + op.on("--timezone=T", + "time zone to apply to zoneless timestamps [default UTC]") { |s| tz = s } + op.on("--memstat-select=RX", "memstat: select memory types matching RX") { |s| + memstat_select = Regexp.new s + } + op.on("--memstat-reject=RX", "memstat: reject memory types matching RX") { |s| + memstat_reject = Regexp.new s + } +end.parse! + + +if format =~ /\Amemstat(?:-(.*))?/ + memstat_type = $1 || 'plain' + unless %w[plain human json].include? memstat_type + raise "unknown memstat type #{memstat_type.dump}" + end + format = 'memstat' +end + +repr, logsep = case format +when 'yaml' + require 'yaml' + + [proc { |e| e.to_yaml }, "\n"] +when 'json', 'memstat' + require 'json' + + [proc { |e| e.to_json }, " "] +else + raise "unkonwn format '#{format}'" +end +formatter = proc { |e| puts repr.call(e) } + +INPUT_FORMATS.include? input_format or raise "unkwown input format '#{input_format}'" + +dumpinfo = {} + +# parse a statedump entry +elem_cbk = proc { |s,&cbk| + arraylike = false + s.grep(/\S/).empty? and next + head = nil + while s.last =~ /^\s*$/ + s.pop + end + body = catch { |misc2| + s[0] =~ HEAD ? (head = $1) : (throw misc2) + body = [[]] + s[1..-1].each { |l| + if l =~ ARRSEP + arraylike = true + body << [] + next + end + body.last << l + } + + body.reject(&:empty?).map { |e| + ea = e.map { |l| + k,v = l.split("=",2) + m = /\A(0|-?[1-9]\d*)(\.\d+)?\Z/.match v + [k, m ? (m[2] ? Float(v) : Integer(v)) : v] + } + begin + ea.to_h + rescue + throw misc2 + end + } + } + + if body + cbk.call [head, arraylike ? body : (body.empty? ? {} : body[0])] + else + STDERR.puts ["WARNING: failed to parse record:", repr.call(s)].join(logsep) + end +} + +# aggregator routine +aggr = case format +when 'memstat' + meminfo = {} + # commit memory-related entries to meminfo + proc { |k,r| + case k + when /memusage/ + (meminfo["GF_MALLOC"]||={})[k] ||= r["size"] if k =~ memstat_select and k !~ memstat_reject + when "mempool" + r.each {|e| + kk = "mempool:#{e['pool-name']}" + (meminfo["mempool"]||={})[kk] ||= e["size"] if kk =~ memstat_select and kk !~ memstat_reject + } + end + } +else + # just format data, don't actually aggregate anything + proc { |pair| formatter.call pair } +end + +# processing the data +case input_format +when 'statedump' + acc = [] + $<.each { |l| + l = l.strip + if l =~ /^(DUMP-(?:START|END)-TIME):\s+(.*)/ + dumpinfo["_meta"]||={} + (dumpinfo["_meta"]["date"]||={})[$1] = Time.parse([$2, tz].join " ") + next + end + + if l =~ HEAD + elem_cbk.call(acc, &aggr) + acc = [l] + next + end + + acc << l + } + elem_cbk.call(acc, &aggr) +when 'json' + $<.each { |l| + r = JSON.load l + case r + when Array + aggr[r] + when Hash + dumpinfo.merge! r + end + } +end + +# final actions: output aggregated data +case format +when 'memstat' + ma = meminfo.values.map(&:to_a).inject(:+) + totals = meminfo.map { |coll,h| [coll, h.values.inject(:+)] }.to_h + tt = ma.transpose[1].inject(:+) + + summary_sep,showm = case memstat_type + when 'json' + ["", proc { |k,v| puts({type: k, value: v}.to_json) }] + when 'plain', 'human' + # human-friendly number representation + hr = proc { |n| + qa = %w[B kB MB GB] + q = ((1...qa.size).find {|i| n < (1 << i*10)} || qa.size) - 1 + "%.2f%s" % [n.to_f / (1 << q*10), qa[q]] + } + + templ = "%{val} %{key}" + tft = proc { |t| t } + nft = if memstat_type == 'human' + nw = [ma.transpose[1], totals.values, tt].flatten.map{|n| hr[n].size}.max + proc { |n| + hn = hr[n] + " " * (nw - hn.size) + hn + } + else + nw = tt.to_s.size + proc { |n| "%#{nw}d" % n } + end + ## Alternative template, key first: + # templ = "%{key} %{val}" + # tw = ma.transpose[0].map(&:size).max + # tft = proc { |t| t + " " * [tw - t.size, 0].max } + # nft = (memstat_type == 'human') ? hr : proc { |n| n } + ["\n", proc { |k,v| puts templ % {key: tft[k], val: nft[v]} }] + else + raise 'this should be impossible' + end + + ma.sort_by { |k,v| v }.each(&showm) + print summary_sep + totals.each { |coll,t| showm.call "Total #{coll}", t } + showm.call "TOTAL", tt +else + formatter.call dumpinfo +end diff --git a/extras/stop-all-gluster-processes.sh b/extras/stop-all-gluster-processes.sh index ea2b48ee88b..710aaf5fd3c 100755 --- a/extras/stop-all-gluster-processes.sh +++ b/extras/stop-all-gluster-processes.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # # Kill all the processes/services except glusterd # diff --git a/extras/stripe-merge.c b/extras/stripe-merge.c index 74bd47e303e..e013a6e6e8a 100644 --- a/extras/stripe-merge.c +++ b/extras/stripe-merge.c @@ -28,7 +28,7 @@ #include <stdint.h> #include <errno.h> #include <string.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <fnmatch.h> #define ATTRNAME_STRIPE_INDEX "trusted.*.stripe-index" @@ -40,33 +40,33 @@ #define INVALID_MODE UINT32_MAX struct file_stripe_info { - int stripe_count; - int stripe_size; - int coalesce; - mode_t mode; - int fd[0]; + int stripe_count; + int stripe_size; + int coalesce; + mode_t mode; + int fd[0]; }; -static int close_files(struct file_stripe_info *); +static int +close_files(struct file_stripe_info *); -static struct -file_stripe_info *alloc_file_stripe_info(int count) +static struct file_stripe_info * +alloc_file_stripe_info(int count) { - int i; - struct file_stripe_info *finfo; + int i; + struct file_stripe_info *finfo; - finfo = calloc(1, sizeof(struct file_stripe_info) + - (sizeof(int) * count)); - if (!finfo) - return NULL; + finfo = calloc(1, sizeof(struct file_stripe_info) + (sizeof(int) * count)); + if (!finfo) + return NULL; - for (i = 0; i < count; i++) - finfo->fd[i] = INVALID_FD; + for (i = 0; i < count; i++) + finfo->fd[i] = INVALID_FD; - finfo->mode = INVALID_MODE; - finfo->coalesce = INVALID_FD; + finfo->mode = INVALID_MODE; + finfo->coalesce = INVALID_FD; - return finfo; + return finfo; } /* @@ -77,39 +77,39 @@ file_stripe_info *alloc_file_stripe_info(int count) static int get_stripe_attr_name(const char *path, const char *pattern, char **attrname) { - char attrbuf[4096]; - char *ptr, *match = NULL; - int len, r, match_count = 0; - - if (!path || !pattern || !attrname) - return -1; - - len = listxattr(path, attrbuf, sizeof(attrbuf)); - if (len < 0) - return len; - - ptr = attrbuf; - while (ptr) { - r = fnmatch(pattern, ptr, 0); - if (!r) { - if (!match) - match = ptr; - match_count++; - } else if (r != FNM_NOMATCH) { - return -1; - } - - len -= strlen(ptr) + 1; - if (len > 0) - ptr += strlen(ptr) + 1; - else - ptr = NULL; - } - - if (match) - *attrname = strdup(match); - - return match_count; + char attrbuf[4096]; + char *ptr, *match = NULL; + int len, r, match_count = 0; + + if (!path || !pattern || !attrname) + return -1; + + len = listxattr(path, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; + + ptr = attrbuf; + while (ptr) { + r = fnmatch(pattern, ptr, 0); + if (!r) { + if (!match) + match = ptr; + match_count++; + } else if (r != FNM_NOMATCH) { + return -1; + } + + len -= strlen(ptr) + 1; + if (len > 0) + ptr += strlen(ptr) + 1; + else + ptr = NULL; + } + + if (match) + *attrname = strdup(match); + + return match_count; } /* @@ -118,19 +118,19 @@ get_stripe_attr_name(const char *path, const char *pattern, char **attrname) static int get_stripe_attr_val(const char *path, const char *attr, int *val) { - char attrbuf[4096]; - int len; + char attrbuf[4096]; + int len; - if (!path || !attr || !val) - return -1; + if (!path || !attr || !val) + return -1; - len = getxattr(path, attr, attrbuf, sizeof(attrbuf)); - if (len < 0) - return len; + len = getxattr(path, attr, attrbuf, sizeof(attrbuf)); + if (len < 0) + return len; - *val = atoi(attrbuf); + *val = atoi(attrbuf); - return 0; + return 0; } /* @@ -145,29 +145,31 @@ get_stripe_attr_val(const char *path, const char *attr, int *val) static int get_attr(const char *path, const char *pattern, char **buf, int *val) { - int count = 1; - - if (!buf) - return -1; - - if (!*buf) { - count = get_stripe_attr_name(path, pattern, buf); - if (count > 1) { - /* pattern isn't good enough */ - fprintf(stderr, "ERROR: duplicate attributes found " - "matching pattern: %s\n", pattern); - free(*buf); - *buf = NULL; - return count; - } else if (count < 1) { - return count; - } - } - - if (get_stripe_attr_val(path, *buf, val) < 0) - return -1; - - return count; + int count = 1; + + if (!buf) + return -1; + + if (!*buf) { + count = get_stripe_attr_name(path, pattern, buf); + if (count > 1) { + /* pattern isn't good enough */ + fprintf(stderr, + "ERROR: duplicate attributes found " + "matching pattern: %s\n", + pattern); + free(*buf); + *buf = NULL; + return count; + } else if (count < 1) { + return count; + } + } + + if (get_stripe_attr_val(path, *buf, val) < 0) + return -1; + + return count; } /* @@ -178,164 +180,168 @@ get_attr(const char *path, const char *pattern, char **buf, int *val) * print a warning if any files are missing. We proceed without error in the * latter case to support partial recovery. */ -static struct -file_stripe_info *validate_and_open_files(char *paths[], int count) +static struct file_stripe_info * +validate_and_open_files(char *paths[], int count) { - int i, val, tmp; - struct stat sbuf; - char *stripe_count_attr = NULL; - char *stripe_size_attr = NULL; - char *stripe_index_attr = NULL; - char *stripe_coalesce_attr = NULL; - struct file_stripe_info *finfo = NULL; - - for (i = 0; i < count; i++) { - if (!paths[i]) - goto err; - - /* - * Check the stripe count first so we can allocate the info - * struct with the appropriate number of fds. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, - &stripe_count_attr, &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_COUNT); - goto err; - } - if (!finfo) { - finfo = alloc_file_stripe_info(val); - if (!finfo) - goto err; - - if (val != count) - fprintf(stderr, "WARNING: %s: stripe-count " - "(%d) != file count (%d). Result may " - "be incomplete.\n", paths[i], val, - count); - - finfo->stripe_count = val; - } else if (val != finfo->stripe_count) { - fprintf(stderr, "ERROR %s: invalid stripe count: %d " - "(expected %d)\n", paths[i], val, - finfo->stripe_count); - goto err; - } - - /* - * Get and validate the chunk size. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, - &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_SIZE); - goto err; - } - - if (!finfo->stripe_size) { - finfo->stripe_size = val; - } else if (val != finfo->stripe_size) { - fprintf(stderr, "ERROR: %s: invalid stripe size: %d " - "(expected %d)\n", paths[i], val, - finfo->stripe_size); - goto err; - } - - /* - * stripe-coalesce is a backward compatible attribute. If the - * attribute does not exist, assume a value of zero for the - * traditional stripe format. - */ - tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE, - &stripe_coalesce_attr, &val); - if (!tmp) { - val = 0; - } else if (tmp != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_COALESCE); - goto err; - } - - if (finfo->coalesce == INVALID_FD) { - finfo->coalesce = val; - } else if (val != finfo->coalesce) { - fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", - paths[i]); - goto err; - } - - /* - * Get/validate the stripe index and open the file in the - * appropriate fd slot. - */ - if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, - &stripe_index_attr, &val) != 1) { - fprintf(stderr, "ERROR: %s: attribute: '%s'\n", - paths[i], ATTRNAME_STRIPE_INDEX); - goto err; - } - if (finfo->fd[val] != INVALID_FD) { - fprintf(stderr, "ERROR: %s: duplicate stripe index: " - "%d\n", paths[i], val); - goto err; - } - - finfo->fd[val] = open(paths[i], O_RDONLY); - if (finfo->fd[val] < 0) - goto err; - - /* - * Get the creation mode for the file. - */ - if (fstat(finfo->fd[val], &sbuf) < 0) - goto err; - if (finfo->mode == INVALID_MODE) { - finfo->mode = sbuf.st_mode; - } else if (sbuf.st_mode != finfo->mode) { - fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]); - goto err; - } - } - - free(stripe_count_attr); - free(stripe_size_attr); - free(stripe_index_attr); - free(stripe_coalesce_attr); - - return finfo; + int i, val, tmp; + struct stat sbuf; + char *stripe_count_attr = NULL; + char *stripe_size_attr = NULL; + char *stripe_index_attr = NULL; + char *stripe_coalesce_attr = NULL; + struct file_stripe_info *finfo = NULL; + + for (i = 0; i < count; i++) { + if (!paths[i]) + goto err; + + /* + * Check the stripe count first so we can allocate the info + * struct with the appropriate number of fds. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_COUNT, &stripe_count_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COUNT); + goto err; + } + if (!finfo) { + finfo = alloc_file_stripe_info(val); + if (!finfo) + goto err; + + if (val != count) + fprintf(stderr, + "WARNING: %s: stripe-count " + "(%d) != file count (%d). Result may " + "be incomplete.\n", + paths[i], val, count); + + finfo->stripe_count = val; + } else if (val != finfo->stripe_count) { + fprintf(stderr, + "ERROR %s: invalid stripe count: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_count); + goto err; + } + + /* + * Get and validate the chunk size. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_SIZE, &stripe_size_attr, &val) != + 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_SIZE); + goto err; + } + + if (!finfo->stripe_size) { + finfo->stripe_size = val; + } else if (val != finfo->stripe_size) { + fprintf(stderr, + "ERROR: %s: invalid stripe size: %d " + "(expected %d)\n", + paths[i], val, finfo->stripe_size); + goto err; + } + + /* + * stripe-coalesce is a backward compatible attribute. If the + * attribute does not exist, assume a value of zero for the + * traditional stripe format. + */ + tmp = get_attr(paths[i], ATTRNAME_STRIPE_COALESCE, + &stripe_coalesce_attr, &val); + if (!tmp) { + val = 0; + } else if (tmp != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_COALESCE); + goto err; + } + + if (finfo->coalesce == INVALID_FD) { + finfo->coalesce = val; + } else if (val != finfo->coalesce) { + fprintf(stderr, "ERROR: %s: invalid coalesce flag\n", paths[i]); + goto err; + } + + /* + * Get/validate the stripe index and open the file in the + * appropriate fd slot. + */ + if (get_attr(paths[i], ATTRNAME_STRIPE_INDEX, &stripe_index_attr, + &val) != 1) { + fprintf(stderr, "ERROR: %s: attribute: '%s'\n", paths[i], + ATTRNAME_STRIPE_INDEX); + goto err; + } + if (finfo->fd[val] != INVALID_FD) { + fprintf(stderr, + "ERROR: %s: duplicate stripe index: " + "%d\n", + paths[i], val); + goto err; + } + + finfo->fd[val] = open(paths[i], O_RDONLY); + if (finfo->fd[val] < 0) + goto err; + + /* + * Get the creation mode for the file. + */ + if (fstat(finfo->fd[val], &sbuf) < 0) + goto err; + if (finfo->mode == INVALID_MODE) { + finfo->mode = sbuf.st_mode; + } else if (sbuf.st_mode != finfo->mode) { + fprintf(stderr, "ERROR: %s: invalid mode\n", paths[i]); + goto err; + } + } + + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); + + return finfo; err: - free(stripe_count_attr); - free(stripe_size_attr); - free(stripe_index_attr); - free(stripe_coalesce_attr); + free(stripe_count_attr); + free(stripe_size_attr); + free(stripe_index_attr); + free(stripe_coalesce_attr); - if (finfo) { - close_files(finfo); - free(finfo); - } + if (finfo) { + close_files(finfo); + free(finfo); + } - return NULL; + return NULL; } static int close_files(struct file_stripe_info *finfo) { - int i, ret; + int i, ret; - if (!finfo) - return -1; + if (!finfo) + return -1; - for (i = 0; i < finfo->stripe_count; i++) { - if (finfo->fd[i] == INVALID_FD) - continue; + for (i = 0; i < finfo->stripe_count; i++) { + if (finfo->fd[i] == INVALID_FD) + continue; - ret = close(finfo->fd[i]); - if (ret < 0) - return ret; - } + ret = close(finfo->fd[i]); + if (ret < 0) + return ret; + } - return ret; + return ret; } /* @@ -351,43 +357,43 @@ close_files(struct file_stripe_info *finfo) static int generate_file_coalesce(int target, struct file_stripe_info *finfo) { - char *buf; - int ret = 0; - int r, w, i; - - buf = malloc(finfo->stripe_size); - if (!buf) - return -1; - - i = 0; - while (1) { - if (finfo->fd[i] == INVALID_FD) { - if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0) - break; - - i = (i + 1) % finfo->stripe_count; - continue; - } - - r = read(finfo->fd[i], buf, finfo->stripe_size); - if (r < 0) { - ret = r; - break; - } - if (!r) - break; - - w = write(target, buf, r); - if (w < 0) { - ret = w; - break; - } - - i = (i + 1) % finfo->stripe_count; - } - - free(buf); - return ret; + char *buf; + int ret = 0; + int r, w, i; + + buf = malloc(finfo->stripe_size); + if (!buf) + return -1; + + i = 0; + while (1) { + if (finfo->fd[i] == INVALID_FD) { + if (lseek(target, finfo->stripe_size, SEEK_CUR) < 0) + break; + + i = (i + 1) % finfo->stripe_count; + continue; + } + + r = read(finfo->fd[i], buf, finfo->stripe_size); + if (r < 0) { + ret = r; + break; + } + if (!r) + break; + + w = write(target, buf, r); + if (w < 0) { + ret = w; + break; + } + + i = (i + 1) % finfo->stripe_count; + } + + free(buf); + return ret; } /* @@ -398,97 +404,100 @@ generate_file_coalesce(int target, struct file_stripe_info *finfo) static int generate_file_traditional(int target, struct file_stripe_info *finfo) { - int i, j, max_ret, ret; - char buf[finfo->stripe_count][4096]; - - do { - char newbuf[4096] = {0, }; - - max_ret = 0; - for (i = 0; i < finfo->stripe_count; i++) { - memset(buf[i], 0, 4096); - ret = read(finfo->fd[i], buf[i], 4096); - if (ret > max_ret) - max_ret = ret; - } - for (i = 0; i < max_ret; i++) - for (j = 0; j < finfo->stripe_count; j++) - newbuf[i] |= buf[j][i]; - write(target, newbuf, max_ret); - } while (max_ret); - - return 0; + int i, j, max_ret, ret; + char buf[finfo->stripe_count][4096]; + + do { + char newbuf[4096] = { + 0, + }; + + max_ret = 0; + for (i = 0; i < finfo->stripe_count; i++) { + memset(buf[i], 0, 4096); + ret = read(finfo->fd[i], buf[i], 4096); + if (ret > max_ret) + max_ret = ret; + } + for (i = 0; i < max_ret; i++) + for (j = 0; j < finfo->stripe_count; j++) + newbuf[i] |= buf[j][i]; + write(target, newbuf, max_ret); + } while (max_ret); + + return 0; } static int generate_file(int target, struct file_stripe_info *finfo) { - if (finfo->coalesce) - return generate_file_coalesce(target, finfo); + if (finfo->coalesce) + return generate_file_coalesce(target, finfo); - return generate_file_traditional(target, finfo); + return generate_file_traditional(target, finfo); } static void usage(char *name) { - fprintf(stderr, "Usage: %s [-o <outputfile>] <inputfile1> " - "<inputfile2> ...\n", name); + fprintf(stderr, + "Usage: %s [-o <outputfile>] <inputfile1> " + "<inputfile2> ...\n", + name); } int main(int argc, char *argv[]) { - int file_count, opt; - char *opath = NULL; - int targetfd; - struct file_stripe_info *finfo; - - while ((opt = getopt(argc, argv, "o:")) != -1) { - switch (opt) { - case 'o': - opath = optarg; - break; - default: - usage(argv[0]); - return -1; - } - } - - file_count = argc - optind; - - if (!opath || !file_count) { - usage(argv[0]); - return -1; - } - - finfo = validate_and_open_files(&argv[optind], file_count); - if (!finfo) - goto err; - - targetfd = open(opath, O_RDWR|O_CREAT, finfo->mode); - if (targetfd < 0) - goto err; - - if (generate_file(targetfd, finfo) < 0) - goto err; - - if (fsync(targetfd) < 0) - fprintf(stderr, "ERROR: %s\n", strerror(errno)); - if (close(targetfd) < 0) - fprintf(stderr, "ERROR: %s\n", strerror(errno)); - - close_files(finfo); - free(finfo); - - return 0; + int file_count, opt; + char *opath = NULL; + int targetfd; + struct file_stripe_info *finfo; + + while ((opt = getopt(argc, argv, "o:")) != -1) { + switch (opt) { + case 'o': + opath = optarg; + break; + default: + usage(argv[0]); + return -1; + } + } + + file_count = argc - optind; + + if (!opath || !file_count) { + usage(argv[0]); + return -1; + } + + finfo = validate_and_open_files(&argv[optind], file_count); + if (!finfo) + goto err; + + targetfd = open(opath, O_RDWR | O_CREAT, finfo->mode); + if (targetfd < 0) + goto err; + + if (generate_file(targetfd, finfo) < 0) + goto err; + + if (fsync(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + if (close(targetfd) < 0) + fprintf(stderr, "ERROR: %s\n", strerror(errno)); + + close_files(finfo); + free(finfo); + + return 0; err: - if (finfo) { - close_files(finfo); - free(finfo); - } + if (finfo) { + close_files(finfo); + free(finfo); + } - return -1; + return -1; } - diff --git a/extras/systemd/Makefile.am b/extras/systemd/Makefile.am index 3988b40bce6..61446a9b84a 100644 --- a/extras/systemd/Makefile.am +++ b/extras/systemd/Makefile.am @@ -1,11 +1,17 @@ -CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service -EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in +CLEANFILES = glusterd.service glustereventsd.service glusterfssharedstorage.service gluster-ta-volume.service +EXTRA_DIST = glusterd.service.in glustereventsd.service.in glusterfssharedstorage.service.in gluster-ta-volume.service.in if USE_SYSTEMD +systemd_DATA = gluster-ta-volume.service +endif + +if WITH_SERVER +if USE_SYSTEMD # systemddir is already defined through configure.ac -systemd_DATA = glusterd.service glusterfssharedstorage.service +systemd_DATA += glusterd.service glusterfssharedstorage.service if BUILD_EVENTS systemd_DATA += glustereventsd.service endif endif +endif diff --git a/extras/systemd/gluster-ta-volume.service.in b/extras/systemd/gluster-ta-volume.service.in new file mode 100644 index 00000000000..2802bca05bf --- /dev/null +++ b/extras/systemd/gluster-ta-volume.service.in @@ -0,0 +1,13 @@ +[Unit] +Description=GlusterFS, Thin-arbiter process to maintain quorum for replica volume +After=network.target + +[Service] +Environment="LOG_LEVEL=WARNING" +ExecStart=@prefix@/sbin/glusterfsd -N --volfile-id ta -f @GLUSTERD_WORKDIR@/thin-arbiter/thin-arbiter.vol --brick-port 24007 --xlator-option ta-server.transport.socket.listen-port=24007 -LWARNING +Restart=always +KillMode=process +SuccessExitStatus=15 + +[Install] +WantedBy=multi-user.target diff --git a/extras/systemd/glusterd.service.in b/extras/systemd/glusterd.service.in index 78f8e40b0e4..abb0d82911f 100644 --- a/extras/systemd/glusterd.service.in +++ b/extras/systemd/glusterd.service.in @@ -1,7 +1,10 @@ [Unit] Description=GlusterFS, a clustered file-system server -Requires=rpcbind.service -After=network.target rpcbind.service +Documentation=man:glusterd(8) +StartLimitBurst=6 +StartLimitIntervalSec=3600 +Requires=@RPCBIND_SERVICE@ +After=network.target @RPCBIND_SERVICE@ Before=network-online.target [Service] @@ -9,10 +12,15 @@ Type=forking PIDFile=@localstatedir@/run/glusterd.pid LimitNOFILE=65536 Environment="LOG_LEVEL=INFO" -EnvironmentFile=-@sysconfdir@/sysconfig/glusterd +EnvironmentFile=-@SYSCONF_DIR@/sysconfig/glusterd ExecStart=@prefix@/sbin/glusterd -p @localstatedir@/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS KillMode=process +TimeoutSec=300 SuccessExitStatus=15 +Restart=on-abnormal +RestartSec=60 +StartLimitBurst=6 +StartLimitInterval=3600 [Install] WantedBy=multi-user.target diff --git a/extras/systemd/glustereventsd.service.in b/extras/systemd/glustereventsd.service.in index 4bfcf42f386..f80b78199f6 100644 --- a/extras/systemd/glustereventsd.service.in +++ b/extras/systemd/glustereventsd.service.in @@ -1,6 +1,8 @@ [Unit] Description=Gluster Events Notifier -After=syslog.target network.target +After=network.target +Documentation=man:glustereventsd(8) + [Service] Environment=PYTHONPATH=@BUILD_PYTHON_SITE_PACKAGES_EXPANDED@:$PYTHONPATH diff --git a/extras/test/ld-preload-test/ld-preload-lib.c b/extras/test/ld-preload-test/ld-preload-lib.c index 8f74a25cf68..d120c053a69 100644 --- a/extras/test/ld-preload-test/ld-preload-lib.c +++ b/extras/test/ld-preload-test/ld-preload-lib.c @@ -34,594 +34,582 @@ #include <fcntl.h> #include <sys/stat.h> #include <dirent.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> /* Err number that is assigned to errno so that test application can * verify that the function was intercepted correctly. */ -#define PRELOAD_ERRNO_VERF 6449 -#define set_errno() (errno = PRELOAD_ERRNO_VERF) +#define PRELOAD_ERRNO_VERF 6449 +#define set_errno() (errno = PRELOAD_ERRNO_VERF) void -intercept (char *call, int tabs) +intercept(char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } - fprintf (stdout, "Intercepted by %s", call); + fprintf(stdout, "Intercepted by %s", call); } int -creat64 (const char *pathname, mode_t mode) +creat64(const char *pathname, mode_t mode) { - intercept ("creat64", 2); - set_errno (); - return -1; + intercept("creat64", 2); + set_errno(); + return -1; } int -creat (const char *pathname, mode_t mode) +creat(const char *pathname, mode_t mode) { - intercept ("creat", 2); - set_errno (); - return -1; + intercept("creat", 2); + set_errno(); + return -1; } - int -close (int fd) +close(int fd) { - intercept ("close", 2); - set_errno (); - return -1; + intercept("close", 2); + set_errno(); + return -1; } int -open64 (const char *pathname, int flags, ...) +open64(const char *pathname, int flags, ...) { - intercept ("open64", 2); - set_errno (); - return -1; + intercept("open64", 2); + set_errno(); + return -1; } - int -open (const char *pathname, int flags, ...) +open(const char *pathname, int flags, ...) { - intercept ("open", 2); - set_errno (); - return -1; + intercept("open", 2); + set_errno(); + return -1; } ssize_t -read (int fd, void *buf, size_t count) +read(int fd, void *buf, size_t count) { - intercept ("read", 2); - set_errno (); - return -1; + intercept("read", 2); + set_errno(); + return -1; } ssize_t -readv (int fd, const struct iovec *vector, int count) +readv(int fd, const struct iovec *vector, int count) { - intercept ("readv", 2); - set_errno (); - return -1; + intercept("readv", 2); + set_errno(); + return -1; } ssize_t -pread (int fd, void *buf, size_t count, unsigned long offset) +pread(int fd, void *buf, size_t count, unsigned long offset) { - intercept ("pread", 2); - set_errno (); - return -1; + intercept("pread", 2); + set_errno(); + return -1; } - ssize_t -pread64 (int fd, void *buf, size_t count, uint64_t offset) +pread64(int fd, void *buf, size_t count, uint64_t offset) { - intercept ("pread64", 2); - set_errno (); - return -1; + intercept("pread64", 2); + set_errno(); + return -1; } ssize_t -write (int fd, const void *buf, size_t count) +write(int fd, const void *buf, size_t count) { - intercept ("write", 2); - set_errno (); - return -1; + intercept("write", 2); + set_errno(); + return -1; } ssize_t -writev (int fd, const struct iovec *vector, int count) +writev(int fd, const struct iovec *vector, int count) { - intercept ("writev", 2); - set_errno (); - return -1; + intercept("writev", 2); + set_errno(); + return -1; } ssize_t -pwrite (int fd, const void *buf, size_t count, unsigned long offset) +pwrite(int fd, const void *buf, size_t count, unsigned long offset) { - intercept ("pwrite", 2); - set_errno (); - return -1; + intercept("pwrite", 2); + set_errno(); + return -1; } ssize_t -pwrite64 (int fd, const void *buf, size_t count, uint64_t offset) +pwrite64(int fd, const void *buf, size_t count, uint64_t offset) { - intercept ("pwrite64", 2); - set_errno (); - return -1; + intercept("pwrite64", 2); + set_errno(); + return -1; } - off_t -lseek (int fildes, unsigned long offset, int whence) +lseek(int fildes, unsigned long offset, int whence) { - intercept ("lseek", 2); - set_errno (); - return -1; + intercept("lseek", 2); + set_errno(); + return -1; } off_t -lseek64 (int fildes, uint64_t offset, int whence) +lseek64(int fildes, uint64_t offset, int whence) { - intercept ("lseek64", 2); - set_errno (); - return -1; + intercept("lseek64", 2); + set_errno(); + return -1; } - int -dup (int fd) +dup(int fd) { - intercept ("dup", 2); - set_errno (); - return -1; + intercept("dup", 2); + set_errno(); + return -1; } int -dup2 (int oldfd, int newfd) +dup2(int oldfd, int newfd) { - intercept ("dup2", 2); - set_errno (); - return -1; + intercept("dup2", 2); + set_errno(); + return -1; } int -mkdir (const char *pathname, mode_t mode) +mkdir(const char *pathname, mode_t mode) { - intercept ("mkdir", 2); - set_errno (); - return -1; + intercept("mkdir", 2); + set_errno(); + return -1; } int -rmdir (const char *pathname) +rmdir(const char *pathname) { - intercept ("rmdir", 2); - set_errno (); - return -1; + intercept("rmdir", 2); + set_errno(); + return -1; } int -chmod (const char *pathname, mode_t mode) +chmod(const char *pathname, mode_t mode) { - intercept ("chmod", 2); - set_errno (); - return -1; + intercept("chmod", 2); + set_errno(); + return -1; } int -chown (const char *pathname, uid_t owner, gid_t group) +chown(const char *pathname, uid_t owner, gid_t group) { - intercept ("chown", 2); - set_errno (); - return -1; + intercept("chown", 2); + set_errno(); + return -1; } int -fchmod (int fd, mode_t mode) +fchmod(int fd, mode_t mode) { - intercept ("fchmod", 2); - set_errno (); - return -1; + intercept("fchmod", 2); + set_errno(); + return -1; } int -fchown (int fd, uid_t uid, gid_t gid) +fchown(int fd, uid_t uid, gid_t gid) { - intercept ("fchown", 2); - set_errno (); - return -1; + intercept("fchown", 2); + set_errno(); + return -1; } -int fsync (int fd) +int +fsync(int fd) { - intercept ("fsync", 2); - set_errno (); - return -1; + intercept("fsync", 2); + set_errno(); + return -1; } - int -ftruncate (int fd, off_t length) +ftruncate(int fd, off_t length) { - intercept ("ftruncate", 1); - set_errno (); - return -1; + intercept("ftruncate", 1); + set_errno(); + return -1; } - int -ftruncate64 (int fd, off_t length) +ftruncate64(int fd, off_t length) { - intercept ("ftruncate64", 1); - set_errno (); - return -1; + intercept("ftruncate64", 1); + set_errno(); + return -1; } int -link (const char *oldpath, const char *newname) +link(const char *oldpath, const char *newname) { - intercept ("link", 2); - set_errno (); - return -1; + intercept("link", 2); + set_errno(); + return -1; } int -rename (const char *oldpath, const char *newpath) +rename(const char *oldpath, const char *newpath) { - intercept ("rename", 2); - set_errno (); - return -1; + intercept("rename", 2); + set_errno(); + return -1; } int -utimes (const char *path, const struct timeval times[2]) +utimes(const char *path, const struct timeval times[2]) { - intercept ("utimes", 2); - set_errno (); - return -1; + intercept("utimes", 2); + set_errno(); + return -1; } int -utime (const char *path, const struct utimbuf *buf) +futimes(int fd, const struct timeval times[2]) { - intercept ("utime", 2); - set_errno (); - return -1; + intercept("futimes", 2); + set_errno(); + return -1; } - int -mknod (const char *path, mode_t mode, dev_t dev) +utime(const char *path, const struct utimbuf *buf) { - intercept ("mknod", 2); - set_errno (); - return -1; + intercept("utime", 2); + set_errno(); + return -1; } int -__xmknod (int ver, const char *path, mode_t mode, dev_t *dev) +mknod(const char *path, mode_t mode, dev_t dev) { - intercept ("__xmknod", 2); - set_errno (); - return -1; + intercept("mknod", 2); + set_errno(); + return -1; } int -mkfifo (const char *path, mode_t mode) +__xmknod(int ver, const char *path, mode_t mode, dev_t *dev) { - intercept ("mkfifo", 2); - set_errno (); - return -1; + intercept("__xmknod", 2); + set_errno(); + return -1; } int -unlink (const char *path) +mkfifo(const char *path, mode_t mode) { - intercept ("unlink", 2); - set_errno (); - return -1; + intercept("mkfifo", 2); + set_errno(); + return -1; } - int -symlink (const char *oldpath, const char *newpath) +unlink(const char *path) { - intercept ("symlink", 2); - set_errno (); - return -1; + intercept("unlink", 2); + set_errno(); + return -1; } int -readlink (const char *path, char *buf, size_t bufsize) +symlink(const char *oldpath, const char *newpath) { - intercept ("readlink", 1); - set_errno (); - return -1; + intercept("symlink", 2); + set_errno(); + return -1; } +int +readlink(const char *path, char *buf, size_t bufsize) +{ + intercept("readlink", 1); + set_errno(); + return -1; +} char * -realpath (const char *path, char *resolved) +realpath(const char *path, char *resolved) { - intercept ("realpath", 1); - set_errno (); - return NULL; + intercept("realpath", 1); + set_errno(); + return NULL; } - DIR * -opendir (const char *path) +opendir(const char *path) { - intercept ("opendir", 2); - set_errno (); - return NULL; + intercept("opendir", 2); + set_errno(); + return NULL; } - struct dirent * -readdir (DIR *dir) +readdir(DIR *dir) { - intercept ("readdir\t", 2); - set_errno (); - return NULL; + intercept("readdir\t", 2); + set_errno(); + return NULL; } struct dirent * -readdir64 (DIR *dir) +readdir64(DIR *dir) { - intercept ("readdir64", 2); - set_errno (); - return NULL; + intercept("readdir64", 2); + set_errno(); + return NULL; } - int -readdir_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir_r", 1); - set_errno (); - return -1; + intercept("readdir_r", 1); + set_errno(); + return -1; } int -readdir64_r (DIR *dir, struct dirent *entry, struct dirent **result) +readdir64_r(DIR *dir, struct dirent *entry, struct dirent **result) { - intercept ("readdir64_r", 1); - set_errno (); - return -1; + intercept("readdir64_r", 1); + set_errno(); + return -1; } - int -closedir (DIR *dh) +closedir(DIR *dh) { - intercept ("closedir", 1); - set_errno (); - return -1; + intercept("closedir", 1); + set_errno(); + return -1; } int -__xstat (int ver, const char *path, struct stat *buf) +__xstat(int ver, const char *path, struct stat *buf) { - intercept ("__xstat\t", 2); - set_errno (); - return -1; + intercept("__xstat\t", 2); + set_errno(); + return -1; } - int -__xstat64 (int ver, const char *path, struct stat *buf) +__xstat64(int ver, const char *path, struct stat *buf) { - intercept ("__xstat64", 2); - set_errno (); - return -1; + intercept("__xstat64", 2); + set_errno(); + return -1; } int -stat (const char *path, struct stat *buf) +stat(const char *path, struct stat *buf) { - intercept ("stat", 2); - set_errno (); - return -1; + intercept("stat", 2); + set_errno(); + return -1; } int -stat64 (const char *path, struct stat *buf) +stat64(const char *path, struct stat *buf) { - intercept ("stat64", 2); - set_errno (); - return -1; + intercept("stat64", 2); + set_errno(); + return -1; } int -__fxstat (int ver, int fd, struct stat *buf) +__fxstat(int ver, int fd, struct stat *buf) { - intercept ("__fxstat\t", 2); - set_errno (); - return -1; + intercept("__fxstat\t", 2); + set_errno(); + return -1; } - int -__fxstat64 (int ver, int fd, struct stat *buf) +__fxstat64(int ver, int fd, struct stat *buf) { - intercept ("__fxstat64", 2); - set_errno (); - return -1; + intercept("__fxstat64", 2); + set_errno(); + return -1; } int -fstat (int fd, struct stat *buf) +fstat(int fd, struct stat *buf) { - intercept ("fstat", 2); - set_errno (); - return -1; + intercept("fstat", 2); + set_errno(); + return -1; } int -fstat64 (int fd , struct stat *buf) +fstat64(int fd, struct stat *buf) { - intercept ("fstat64", 2); - set_errno (); - return -1; + intercept("fstat64", 2); + set_errno(); + return -1; } int -__lxstat (int ver, const char *path, struct stat *buf) +__lxstat(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat\t", 2); - set_errno (); - return -1; + intercept("__lxstat\t", 2); + set_errno(); + return -1; } int -__lxstat64 (int ver, const char *path, struct stat *buf) +__lxstat64(int ver, const char *path, struct stat *buf) { - intercept ("__lxstat64", 2); - set_errno (); - return -1; + intercept("__lxstat64", 2); + set_errno(); + return -1; } int -lstat (const char *path, struct stat *buf) +lstat(const char *path, struct stat *buf) { - intercept ("lstat", 2); - set_errno (); - return -1; + intercept("lstat", 2); + set_errno(); + return -1; } int -lstat64 (const char *path, struct stat *buf) +lstat64(const char *path, struct stat *buf) { - intercept ("lstat64", 2); - set_errno (); - return -1; + intercept("lstat64", 2); + set_errno(); + return -1; } int -statfs (const char *path, struct statfs *buf) +statfs(const char *path, struct statfs *buf) { - intercept ("statfs", 2); - set_errno (); - return -1; + intercept("statfs", 2); + set_errno(); + return -1; } - int -statfs64 (const char *path, struct statfs *buf) +statfs64(const char *path, struct statfs *buf) { - intercept ("statfs64", 2); - set_errno (); - return -1; + intercept("statfs64", 2); + set_errno(); + return -1; } int -statvfs (const char *path, struct statvfs *buf) +statvfs(const char *path, struct statvfs *buf) { - intercept ("statvfs\t", 2); - set_errno (); - return -1; + intercept("statvfs\t", 2); + set_errno(); + return -1; } - int -statvfs64 (const char *path, struct statvfs *buf) +statvfs64(const char *path, struct statvfs *buf) { - intercept ("statvfs64", 2); - set_errno (); - return -1; + intercept("statvfs64", 2); + set_errno(); + return -1; } ssize_t -getxattr (const char *path, const char *name, void *value, size_t size) +getxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("getxattr", 1); - set_errno (); - return -1; + intercept("getxattr", 1); + set_errno(); + return -1; } ssize_t -lgetxattr (const char *path, const char *name, void *value, size_t size) +lgetxattr(const char *path, const char *name, void *value, size_t size) { - intercept ("lgetxattr", 1); - set_errno (); - return -1; + intercept("lgetxattr", 1); + set_errno(); + return -1; } - int -remove (const char* path) +remove(const char *path) { - intercept ("remove", 2); - set_errno (); - return -1; + intercept("remove", 2); + set_errno(); + return -1; } int -lchown (const char *path, uid_t owner, gid_t group) +lchown(const char *path, uid_t owner, gid_t group) { - intercept ("lchown", 2); - set_errno (); - return -1; + intercept("lchown", 2); + set_errno(); + return -1; } void -rewinddir (DIR *dirp) +rewinddir(DIR *dirp) { - intercept ("rewinddir", 1); - set_errno (); - return; + intercept("rewinddir", 1); + set_errno(); + return; } void -seekdir (DIR *dirp, off_t offset) +seekdir(DIR *dirp, off_t offset) { - intercept ("seekdir", 2); - set_errno (); - return; + intercept("seekdir", 2); + set_errno(); + return; } off_t -telldir (DIR *dirp) +telldir(DIR *dirp) { - intercept ("telldir", 2); - set_errno (); - return -1; + intercept("telldir", 2); + set_errno(); + return -1; } ssize_t -sendfile (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile\t", 1); - set_errno (); - return -1; + intercept("sendfile\t", 1); + set_errno(); + return -1; } ssize_t -sendfile64 (int out_fd, int in_fd, off_t *offset, size_t count) +sendfile64(int out_fd, int in_fd, off_t *offset, size_t count) { - intercept ("sendfile64", 1); - set_errno (); - return -1; + intercept("sendfile64", 1); + set_errno(); + return -1; } - int -fcntl (int fd, int cmd, ...) +fcntl(int fd, int cmd, ...) { - intercept ("fcntl", 2); - set_errno (); - return -1; + intercept("fcntl", 2); + set_errno(); + return -1; } - diff --git a/extras/test/ld-preload-test/ld-preload-test.c b/extras/test/ld-preload-test/ld-preload-test.c index cf8dd52c3e1..54dde8c7d54 100644 --- a/extras/test/ld-preload-test/ld-preload-test.c +++ b/extras/test/ld-preload-test/ld-preload-test.c @@ -46,322 +46,313 @@ #include <sys/uio.h> #include <utime.h> #include <sys/time.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <sys/sendfile.h> - -#define PRELOAD_ERRNO_VERF 6449 +#define PRELOAD_ERRNO_VERF 6449 void check_err(int ret, char *call, int tabs) { - while (tabs > 0) { - fprintf (stdout, "\t"); - --tabs; - } - if (ret != -1) { - fprintf (stdout, "Not intercepted: %s\n", call); - return; - } - - if (errno != PRELOAD_ERRNO_VERF) { - fprintf (stdout, "Not intercepted: %s: err: %s\n", call, - strerror (errno)); - return; - } + while (tabs > 0) { + fprintf(stdout, "\t"); + --tabs; + } + if (ret != -1) { + fprintf(stdout, "Not intercepted: %s\n", call); + return; + } - fprintf (stdout, "Intercept verified: %s\n", call); + if (errno != PRELOAD_ERRNO_VERF) { + fprintf(stdout, "Not intercepted: %s: err: %s\n", call, + strerror(errno)); return; + } + + fprintf(stdout, "Intercept verified: %s\n", call); + return; } void -usage (FILE *fp) +usage(FILE *fp) { - fprintf (fp, "Usage: ld-preload-test <Options>\n"); - fprintf (fp, "Options\n"); - fprintf (fp, "\t--path\t\tPathname is used as the file/directory" - " created for the test.\n"); - + fprintf(fp, "Usage: ld-preload-test <Options>\n"); + fprintf(fp, "Options\n"); + fprintf(fp, + "\t--path\t\tPathname is used as the file/directory" + " created for the test.\n"); } - int -run_file_tests (char *testfile) +run_file_tests(char *testfile) { - int ret = -1; - struct stat buf; + int ret = -1; + struct stat buf; - assert (testfile); - fprintf (stdout, "Testing creat"); - ret = creat (testfile, S_IRWXU); - check_err (ret, "creat", 2); + assert(testfile); + fprintf(stdout, "Testing creat"); + ret = creat(testfile, S_IRWXU); + check_err(ret, "creat", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing open"); - ret = open (testfile, O_RDONLY); - check_err (ret, "open", 2); + fprintf(stdout, "Testing open"); + ret = open(testfile, O_RDONLY); + check_err(ret, "open", 2); - fprintf (stdout, "Testing read"); - ret = read (0, NULL, 0); - check_err (ret, "read", 2); + fprintf(stdout, "Testing read"); + ret = read(0, NULL, 0); + check_err(ret, "read", 2); - fprintf (stdout, "Testing readv"); - ret = readv (0, NULL, 0); - check_err (ret, "readv", 2); + fprintf(stdout, "Testing readv"); + ret = readv(0, NULL, 0); + check_err(ret, "readv", 2); - fprintf (stdout, "Testing pread"); - ret = pread (0, NULL, 0, 0); - check_err (ret, "pread", 2); + fprintf(stdout, "Testing pread"); + ret = pread(0, NULL, 0, 0); + check_err(ret, "pread", 2); - fprintf (stdout, "Testing write"); - ret = write (0, NULL, 0); - check_err (ret, "write", 2); + fprintf(stdout, "Testing write"); + ret = write(0, NULL, 0); + check_err(ret, "write", 2); - fprintf (stdout, "Testing writev"); - ret = writev (0, NULL, 0); - check_err (ret, "writev", 2); + fprintf(stdout, "Testing writev"); + ret = writev(0, NULL, 0); + check_err(ret, "writev", 2); - fprintf (stdout, "Testing pwrite"); - ret = pwrite (0, NULL, 0, 0); - check_err (ret, "pwrite", 2); + fprintf(stdout, "Testing pwrite"); + ret = pwrite(0, NULL, 0, 0); + check_err(ret, "pwrite", 2); - fprintf (stdout, "Testing lseek"); - ret = lseek (0, 0, 0); - check_err (ret, "lseek", 2); + fprintf(stdout, "Testing lseek"); + ret = lseek(0, 0, 0); + check_err(ret, "lseek", 2); - fprintf (stdout, "Testing dup"); - ret = dup (0); - check_err (ret, "dup", 2); + fprintf(stdout, "Testing dup"); + ret = dup(0); + check_err(ret, "dup", 2); - fprintf (stdout, "Testing dup2"); - ret = dup2 (0, 0); - check_err (ret, "dup2", 2); + fprintf(stdout, "Testing dup2"); + ret = dup2(0, 0); + check_err(ret, "dup2", 2); - fprintf (stdout, "Testing fchmod"); - ret = fchmod (0, 0); - check_err (ret, "fchmod", 2); + fprintf(stdout, "Testing fchmod"); + ret = fchmod(0, 0); + check_err(ret, "fchmod", 2); - fprintf (stdout, "Testing fchown"); - ret = fchown (0, 0, 0); - check_err (ret, "fchown", 2); + fprintf(stdout, "Testing fchown"); + ret = fchown(0, 0, 0); + check_err(ret, "fchown", 2); - fprintf (stdout, "Testing fsync"); - ret = fsync (0); - check_err (ret, "fsync", 2); + fprintf(stdout, "Testing fsync"); + ret = fsync(0); + check_err(ret, "fsync", 2); - fprintf (stdout, "Testing ftruncate"); - ret = ftruncate (0, 0); - check_err (ret, "ftruncate", 1); + fprintf(stdout, "Testing ftruncate"); + ret = ftruncate(0, 0); + check_err(ret, "ftruncate", 1); - fprintf (stdout, "Testing fstat"); - ret = fstat (0, &buf); - check_err (ret, "fstat", 1); + fprintf(stdout, "Testing fstat"); + ret = fstat(0, &buf); + check_err(ret, "fstat", 1); - fprintf (stdout, "Testing sendfile"); - ret = sendfile (0, 0, NULL, 0); - check_err (ret, "sendfile", 1); + fprintf(stdout, "Testing sendfile"); + ret = sendfile(0, 0, NULL, 0); + check_err(ret, "sendfile", 1); - fprintf (stdout, "Testing fcntl"); - ret = fcntl (0, 0, NULL); - check_err (ret, "fcntl", 2); + fprintf(stdout, "Testing fcntl"); + ret = fcntl(0, 0, NULL); + check_err(ret, "fcntl", 2); - fprintf (stdout, "Testing close"); - ret = close (ret); - check_err (ret, "close", 2); + fprintf(stdout, "Testing close"); + ret = close(ret); + check_err(ret, "close", 2); - fprintf (stdout, "Testing remove"); - ret = remove (testfile); - check_err (ret, "remove", 2); + fprintf(stdout, "Testing remove"); + ret = remove(testfile); + check_err(ret, "remove", 2); - return ret; + return ret; } - int -run_attr_tests (char *testfile) +run_attr_tests(char *testfile) { - int ret = -1; - char *res = NULL; - struct stat buf; - struct statfs sbuf; - struct statvfs svbuf; - - assert (testfile); - - fprintf (stdout, "Testing chmod"); - ret = chmod (testfile, 0); - check_err (ret, "chmod", 2); - - fprintf (stdout, "Testing chown"); - ret = chown (testfile, 0, 0); - check_err (ret, "chown", 2); - - fprintf (stdout, "Testing link"); - ret = link (testfile, testfile); - check_err (ret, "link", 2); - - fprintf (stdout, "Testing rename"); - ret = rename (testfile, testfile); - check_err (ret, "rename", 2); - - fprintf (stdout, "Testing utimes"); - ret = utimes (testfile, NULL); - check_err (ret, "utimes", 2); - - fprintf (stdout, "Testing utime"); - ret = utime (testfile, NULL); - check_err (ret, "utime", 2); - - fprintf (stdout, "Testing unlink"); - ret = unlink (testfile); - check_err (ret, "unlink", 2); - - fprintf (stdout, "Testing symlink"); - ret = symlink (testfile, testfile); - check_err (ret, "symlink", 2); - - fprintf (stdout, "Testing readlink"); - ret = readlink (testfile, testfile, 0); - check_err (ret, "readlink", 2); - - fprintf (stdout, "Testing realpath"); - ret = 0; - res = realpath ((const char *)testfile, testfile); - if (!res) - ret = -1; - check_err (ret, "realpath", 2); - - fprintf (stdout, "Testing stat"); - ret = stat (testfile, &buf); - check_err (ret, "stat", 1); - - fprintf (stdout, "Testing lstat"); - ret = lstat (testfile, &buf); - check_err (ret, "lstat", 1); - - fprintf (stdout, "Testing statfs"); - ret = statfs (testfile, &sbuf); - check_err (ret, "statfs", 2); - - fprintf (stdout, "Testing statvfs"); - ret = statvfs (testfile, &svbuf); - check_err (ret, "statvfs", 1); - - fprintf (stdout, "Testing getxattr"); - ret = getxattr (testfile, NULL, NULL, 0); - check_err (ret, "getxattr", 2); - - fprintf (stdout, "Testing lgetxattr"); - ret = lgetxattr (testfile, NULL, NULL, 0); - check_err (ret, "lgetxattr", 1); - - fprintf (stdout, "Testing lchown"); - ret = lchown (testfile, 0, 0); - check_err (ret, "lchown", 2); - return 0; + int ret = -1; + char *res = NULL; + struct stat buf; + struct statfs sbuf; + struct statvfs svbuf; + + assert(testfile); + + fprintf(stdout, "Testing chmod"); + ret = chmod(testfile, 0); + check_err(ret, "chmod", 2); + + fprintf(stdout, "Testing chown"); + ret = chown(testfile, 0, 0); + check_err(ret, "chown", 2); + + fprintf(stdout, "Testing link"); + ret = link(testfile, testfile); + check_err(ret, "link", 2); + + fprintf(stdout, "Testing rename"); + ret = rename(testfile, testfile); + check_err(ret, "rename", 2); + + fprintf(stdout, "Testing utimes"); + ret = utimes(testfile, NULL); + check_err(ret, "utimes", 2); + + fprintf(stdout, "Testing utime"); + ret = utime(testfile, NULL); + check_err(ret, "utime", 2); + + fprintf(stdout, "Testing unlink"); + ret = unlink(testfile); + check_err(ret, "unlink", 2); + + fprintf(stdout, "Testing symlink"); + ret = symlink(testfile, testfile); + check_err(ret, "symlink", 2); + + fprintf(stdout, "Testing readlink"); + ret = readlink(testfile, testfile, 0); + check_err(ret, "readlink", 2); + + fprintf(stdout, "Testing realpath"); + ret = 0; + res = realpath((const char *)testfile, testfile); + if (!res) + ret = -1; + check_err(ret, "realpath", 2); + + fprintf(stdout, "Testing stat"); + ret = stat(testfile, &buf); + check_err(ret, "stat", 1); + + fprintf(stdout, "Testing lstat"); + ret = lstat(testfile, &buf); + check_err(ret, "lstat", 1); + + fprintf(stdout, "Testing statfs"); + ret = statfs(testfile, &sbuf); + check_err(ret, "statfs", 2); + + fprintf(stdout, "Testing statvfs"); + ret = statvfs(testfile, &svbuf); + check_err(ret, "statvfs", 1); + + fprintf(stdout, "Testing getxattr"); + ret = getxattr(testfile, NULL, NULL, 0); + check_err(ret, "getxattr", 2); + + fprintf(stdout, "Testing lgetxattr"); + ret = lgetxattr(testfile, NULL, NULL, 0); + check_err(ret, "lgetxattr", 1); + + fprintf(stdout, "Testing lchown"); + ret = lchown(testfile, 0, 0); + check_err(ret, "lchown", 2); + return 0; } - int -run_dev_tests (char *testfile) +run_dev_tests(char *testfile) { - int ret = -1; + int ret = -1; - assert (testfile); + assert(testfile); - fprintf (stdout, "Testing mknod"); - ret = mknod (testfile, 0, 0); - check_err (ret, "mknod", 2); + fprintf(stdout, "Testing mknod"); + ret = mknod(testfile, 0, 0); + check_err(ret, "mknod", 2); - fprintf (stdout, "Testing mkfifo"); - ret = mkfifo (testfile, 0); - check_err (ret, "mkfifo", 2); - return 0; + fprintf(stdout, "Testing mkfifo"); + ret = mkfifo(testfile, 0); + check_err(ret, "mkfifo", 2); + return 0; } int -run_dir_tests (char *testpath) +run_dir_tests(char *testpath) { - int ret = -1; - DIR *dh = NULL; - struct dirent *dire = NULL; - - assert (testpath); - - fprintf (stdout, "Testing mkdir"); - ret = mkdir (testpath, 0); - check_err (ret, "mkdir", 2); - - fprintf (stdout, "Testing rmdir"); - ret = rmdir (testpath); - check_err (ret, "rmdir", 2); - - fprintf (stdout, "Testing opendir"); - ret = 0; - dh = opendir (testpath); - if (!dh) - ret = -1; - check_err (ret, "opendir", 2); - - fprintf (stdout, "Testing readdir"); - ret = 0; - dire = readdir (dh); - if (!dire) - ret = -1; - check_err (ret, "readdir", 1); - - fprintf (stdout, "Testing readdir_r"); - ret = readdir_r (dh, dire, &dire); - check_err (ret, "readdir_r", 1); - - fprintf (stdout, "Testing rewinddir"); - rewinddir (dh); - check_err (-1, "rewinddir", 1); - - fprintf (stdout, "Testing seekdir"); - seekdir (dh, 0); - check_err (-1, "seekdir", 2); - - fprintf (stdout, "Testing telldir"); - ret = telldir (dh); - check_err (ret, "telldir", 2); - - fprintf (stdout, "Testing closedir"); - ret = closedir (dh); - check_err (ret, "closedir", 2); - return 0; + int ret = -1; + DIR *dh = NULL; + struct dirent *dire = NULL; + + assert(testpath); + + fprintf(stdout, "Testing mkdir"); + ret = mkdir(testpath, 0); + check_err(ret, "mkdir", 2); + + fprintf(stdout, "Testing rmdir"); + ret = rmdir(testpath); + check_err(ret, "rmdir", 2); + + fprintf(stdout, "Testing opendir"); + ret = 0; + dh = opendir(testpath); + if (!dh) + ret = -1; + check_err(ret, "opendir", 2); + + fprintf(stdout, "Testing readdir"); + ret = 0; + dire = readdir(dh); + if (!dire) + ret = -1; + check_err(ret, "readdir", 1); + + fprintf(stdout, "Testing readdir_r"); + ret = readdir_r(dh, dire, &dire); + check_err(ret, "readdir_r", 1); + + fprintf(stdout, "Testing rewinddir"); + rewinddir(dh); + check_err(-1, "rewinddir", 1); + + fprintf(stdout, "Testing seekdir"); + seekdir(dh, 0); + check_err(-1, "seekdir", 2); + + fprintf(stdout, "Testing telldir"); + ret = telldir(dh); + check_err(ret, "telldir", 2); + + fprintf(stdout, "Testing closedir"); + ret = closedir(dh); + check_err(ret, "closedir", 2); + return 0; } - - int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - char *testpath = NULL; - int x = 0; - - for (;x < argc; ++x) { - if (strcmp (argv[x], "--path") == 0) { - testpath = argv[x+1]; - continue; - } + char *testpath = NULL; + int x = 0; + for (; x < argc; ++x) { + if (strcmp(argv[x], "--path") == 0) { + testpath = argv[x + 1]; + continue; } + } - if (!testpath) { - fprintf (stderr, "--path not specified\n"); - usage (stderr); - return -1; - } + if (!testpath) { + fprintf(stderr, "--path not specified\n"); + usage(stderr); + return -1; + } - run_file_tests (testpath); - run_dir_tests (testpath); - run_attr_tests (testpath); - run_dev_tests (testpath); + run_file_tests(testpath); + run_dir_tests(testpath); + run_attr_tests(testpath); + run_dev_tests(testpath); - return 0; + return 0; } - - diff --git a/extras/test/open-fd-tests.c b/extras/test/open-fd-tests.c index 4184079d043..509952b4180 100644 --- a/extras/test/open-fd-tests.c +++ b/extras/test/open-fd-tests.c @@ -4,61 +4,64 @@ #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <errno.h> #include <string.h> int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; - int fd = 0; - char *filename = NULL; - int loop = 0; - struct stat stbuf = {0,}; - char string[1024] = {0,}; + int ret = -1; + int fd = 0; + char *filename = NULL; + int loop = 0; + struct stat stbuf = { + 0, + }; + char string[1024] = { + 0, + }; - if (argc > 1) - filename = argv[1]; + if (argc > 1) + filename = argv[1]; - if (!filename) - filename = "temp-fd-test-file"; + if (!filename) + filename = "temp-fd-test-file"; - fd = open (filename, O_RDWR|O_CREAT|O_TRUNC); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - while (loop < 1000) { - /* Use it as a mechanism to test time delays */ - memset (string, 0, 1024); - scanf ("%s", string); + fd = open(filename, O_RDWR | O_CREAT | O_TRUNC); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } - ret = write (fd, string, strlen (string)); - if (ret != strlen (string)) { - fprintf (stderr, "write failed : %s (%s %d)\n", - strerror (errno), string, loop); - goto out; - } + while (loop < 1000) { + /* Use it as a mechanism to test time delays */ + memset(string, 0, 1024); + scanf("%s", string); - ret = write (fd, "\n", 1); - if (ret != 1) { - fprintf (stderr, "write failed : %s (%d)\n", - strerror (errno), loop); - goto out; - } + ret = write(fd, string, strlen(string)); + if (ret != strlen(string)) { + fprintf(stderr, "write failed : %s (%s %d)\n", strerror(errno), + string, loop); + goto out; + } - loop++; + ret = write(fd, "\n", 1); + if (ret != 1) { + fprintf(stderr, "write failed : %s (%d)\n", strerror(errno), loop); + goto out; } - fprintf (stdout, "finishing the test after %d loops\n", loop); + loop++; + } + + fprintf(stdout, "finishing the test after %d loops\n", loop); - ret = 0; + ret = 0; out: - if (fd) - close (fd); + if (fd) + close(fd); - return ret; + return ret; } diff --git a/extras/test/test-ffop.c b/extras/test/test-ffop.c index 219dd6a2da2..1d9c125db67 100644 --- a/extras/test/test-ffop.c +++ b/extras/test/test-ffop.c @@ -3,777 +3,825 @@ #include <unistd.h> #include <sys/types.h> #include <sys/stat.h> -#include <attr/xattr.h> +#include <sys/xattr.h> #include <errno.h> #include <string.h> #include <dirent.h> -int fd_based_fops_1 (char *filename); //for fd based fops after unlink -int fd_based_fops_2 (char *filename); //for fd based fops before unlink -int dup_fd_based_fops (char *filename); // fops based on fd after dup -int path_based_fops (char *filename); //for fops based on path -int dir_based_fops (char *filename); // for fops which operate on directory -int link_based_fops (char *filename); //for fops which operate in link files (symlinks) -int test_open_modes (char *filename); // to test open syscall with open modes available. -int generic_open_read_write (char *filename, int flag); // generic function which does open write and read. +int +fd_based_fops_1(char *filename); // for fd based fops after unlink +int +fd_based_fops_2(char *filename); // for fd based fops before unlink +int +dup_fd_based_fops(char *filename); // fops based on fd after dup +int +path_based_fops(char *filename); // for fops based on path +int +dir_based_fops(char *filename); // for fops which operate on directory +int +link_based_fops( + char *filename); // for fops which operate in link files (symlinks) +int +test_open_modes( + char *filename); // to test open syscall with open modes available. +int +generic_open_read_write( + char *filename, + int flag); // generic function which does open write and read. int -main (int argc, char *argv[]) +main(int argc, char *argv[]) { - int ret = -1; - char filename[255] = {0,}; - - if (argc > 1) - strcpy(filename, argv[1]); - else - strcpy(filename, "temp-xattr-test-file"); - - ret = fd_based_fops_1 (strcat(filename, "_1")); - if (ret < 0) - fprintf (stderr, "fd based file operation 1 failed\n"); - else - fprintf (stdout, "fd based file operation 1 passed\n"); - - ret = fd_based_fops_2 (strcat(filename, "_2")); - if (ret < 0) - fprintf (stderr, "fd based file operation 2 failed\n"); - else - fprintf (stdout, "fd based file operation 2 passed\n"); - - ret = dup_fd_based_fops (strcat (filename, "_3")); - if (ret < 0) - fprintf (stderr, "dup fd based file operation failed\n"); - else - fprintf (stdout, "dup fd based file operation passed\n"); - - ret = path_based_fops (strcat (filename, "_4")); - if (ret < 0) - fprintf (stderr, "path based file operation failed\n"); - else - fprintf (stdout, "path based file operation passed\n"); - - ret = dir_based_fops (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "directory based file operation failed\n"); - else - fprintf (stdout, "directory based file operation passed\n"); - - ret = link_based_fops (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "link based file operation failed\n"); - else - fprintf (stdout, "link based file operation passed\n"); - - ret = test_open_modes (strcat (filename, "_5")); - if (ret < 0) - fprintf (stderr, "testing modes of 'open' call failed\n"); - else - fprintf (stdout, "testing modes of 'open' call passed\n"); + int ret = -1; + char filename[255] = { + 0, + }; + + if (argc > 1) + strcpy(filename, argv[1]); + else + strcpy(filename, "temp-xattr-test-file"); + + ret = fd_based_fops_1(strcat(filename, "_1")); + if (ret < 0) + fprintf(stderr, "fd based file operation 1 failed\n"); + else + fprintf(stdout, "fd based file operation 1 passed\n"); + + ret = fd_based_fops_2(strcat(filename, "_2")); + if (ret < 0) + fprintf(stderr, "fd based file operation 2 failed\n"); + else + fprintf(stdout, "fd based file operation 2 passed\n"); + + ret = dup_fd_based_fops(strcat(filename, "_3")); + if (ret < 0) + fprintf(stderr, "dup fd based file operation failed\n"); + else + fprintf(stdout, "dup fd based file operation passed\n"); + + ret = path_based_fops(strcat(filename, "_4")); + if (ret < 0) + fprintf(stderr, "path based file operation failed\n"); + else + fprintf(stdout, "path based file operation passed\n"); + + ret = dir_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "directory based file operation failed\n"); + else + fprintf(stdout, "directory based file operation passed\n"); + + ret = link_based_fops(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "link based file operation failed\n"); + else + fprintf(stdout, "link based file operation passed\n"); + + ret = test_open_modes(strcat(filename, "_5")); + if (ret < 0) + fprintf(stderr, "testing modes of 'open' call failed\n"); + else + fprintf(stdout, "testing modes of 'open' call passed\n"); out: - return ret; + return ret; } int -fd_based_fops_1 (char *filename) +fd_based_fops_1(char *filename) { - int fd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - ret = unlink (filename); - if (ret < 0) { - fprintf (stderr, "unlink failed : %s\n", strerror (errno)); - goto out; - } - - strcpy (wstr, "This is my string\n"); - ret = write (fd, wstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - ret = lseek (fd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (fd, rstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = ftruncate (fd, 0); - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - ret = fstat (fd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (fd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (fd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (fd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (fd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (fd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (fd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = 0; + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(fd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; out: - if (fd) - close (fd); + if (fd) + close(fd); - return ret; + return ret; } - int -fd_based_fops_2 (char *filename) +fd_based_fops_2(char *filename) { - int fd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - ret = ftruncate (fd, 0); - - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - strcpy (wstr, "This is my second string\n"); - ret = write (fd, wstr, strlen (wstr)); - if (ret < 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - lseek (fd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (fd, rstr, strlen (wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = fstat (fd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (fd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (fd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (fd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (fd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (fd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (fd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (fd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (fd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } + int fd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + ret = ftruncate(fd, 0); + + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + strcpy(wstr, "This is my second string\n"); + ret = write(fd, wstr, strlen(wstr)); + if (ret < 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + lseek(fd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(fd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = fstat(fd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(fd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(fd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(fd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(fd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(fd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(fd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(fd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(fd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } out: - if (fd) - close (fd); - unlink (filename); + if (fd) + close(fd); + unlink(filename); - return ret; + return ret; } int -path_based_fops (char *filename) +path_based_fops(char *filename) { - int ret = -1; - int fd = 0; - struct stat stbuf = {0,}; - char newfilename[255] = {0,}; - - fd = creat (filename, 0644); - if (fd < 0) { - fprintf (stderr, "creat failed: %s\n", strerror (errno)); - goto out; - } - - ret = truncate (filename, 0); - if (ret < 0) { - fprintf (stderr, "truncate failed: %s\n", strerror (errno)); - goto out; - } - - ret = stat (filename, &stbuf); - if (ret < 0) { - fprintf (stderr, "stat failed: %s\n", strerror (errno)); - goto out; - } - - ret = chmod (filename, 0640); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = chown (filename, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "chown failed: %s\n", strerror (errno)); - goto out; - } - - ret = setxattr (filename, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "setxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = listxattr (filename, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "listxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = getxattr (filename, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "getxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = removexattr (filename, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "removexattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = access (filename, R_OK|W_OK); - if (ret < 0) { - fprintf (stderr, "access failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newfilename, filename); - strcat(newfilename, "_new"); - ret = rename (filename, newfilename); - if (ret < 0) { - fprintf (stderr, "rename failed: %s\n", strerror (errno)); - goto out; - } - unlink (newfilename); + int ret = -1; + int fd = 0; + struct stat stbuf = { + 0, + }; + char newfilename[255] = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + ret = truncate(filename, 0); + if (ret < 0) { + fprintf(stderr, "truncate failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(filename, &stbuf); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(filename, 0640); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(filename, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chown failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(filename, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(filename, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(filename, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(filename, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = access(filename, R_OK | W_OK); + if (ret < 0) { + fprintf(stderr, "access failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newfilename, filename); + strcat(newfilename, "_new"); + ret = rename(filename, newfilename); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + unlink(newfilename); out: - if (fd) - close (fd); + if (fd) + close(fd); - unlink (filename); - return ret; + unlink(filename); + return ret; } int -dup_fd_based_fops (char *filename) +dup_fd_based_fops(char *filename) { - int fd = 0; - int newfd = 0; - int ret = -1; - struct stat stbuf = {0,}; - char wstr[50] = {0,}; - char rstr[50] = {0,}; - - fd = open (filename, O_RDWR|O_CREAT); - if (fd < 0) { - fd = 0; - fprintf (stderr, "open failed : %s\n", strerror (errno)); - goto out; - } - - newfd = dup (fd); - if (newfd < 0) { - ret = -1; - fprintf (stderr, "dup failed: %s\n", strerror (errno)); - goto out; - } - - close (fd); - - strcpy (wstr, "This is my string\n"); - ret = write (newfd, wstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "write failed: %s\n", strerror (errno)); - goto out; - } - - ret = lseek (newfd, 0, SEEK_SET); - if (ret < 0) { - fprintf (stderr, "lseek failed: %s\n", strerror (errno)); - goto out; - } - - ret = read (newfd, rstr, strlen(wstr)); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "read failed: %s\n", strerror (errno)); - goto out; - } - - ret = memcmp (rstr, wstr, strlen (wstr)); - if (ret != 0) { - ret = -1; - fprintf (stderr, "read returning junk\n"); - goto out; - } - - ret = ftruncate (newfd, 0); - if (ret < 0) { - fprintf (stderr, "ftruncate failed : %s\n", strerror (errno)); - goto out; - } - - ret = fstat (newfd, &stbuf); - if (ret < 0) { - fprintf (stderr, "fstat failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchmod (newfd, 0640); - if (ret < 0) { - fprintf (stderr, "fchmod failed : %s\n", strerror (errno)); - goto out; - } - - ret = fchown (newfd, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "fchown failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsync (newfd); - if (ret < 0) { - fprintf (stderr, "fsync failed : %s\n", strerror (errno)); - goto out; - } - - ret = fsetxattr (newfd, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "fsetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fdatasync (newfd); - if (ret < 0) { - fprintf (stderr, "fdatasync failed : %s\n", strerror (errno)); - goto out; - } - - ret = flistxattr (newfd, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "flistxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fgetxattr (newfd, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "fgetxattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = fremovexattr (newfd, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "fremovexattr failed : %s\n", strerror (errno)); - goto out; - } - - ret = 0; + int fd = 0; + int newfd = 0; + int ret = -1; + struct stat stbuf = { + 0, + }; + char wstr[50] = { + 0, + }; + char rstr[50] = { + 0, + }; + + fd = open(filename, O_RDWR | O_CREAT); + if (fd < 0) { + fd = 0; + fprintf(stderr, "open failed : %s\n", strerror(errno)); + goto out; + } + + newfd = dup(fd); + if (newfd < 0) { + ret = -1; + fprintf(stderr, "dup failed: %s\n", strerror(errno)); + goto out; + } + + close(fd); + + strcpy(wstr, "This is my string\n"); + ret = write(newfd, wstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "write failed: %s\n", strerror(errno)); + goto out; + } + + ret = lseek(newfd, 0, SEEK_SET); + if (ret < 0) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + goto out; + } + + ret = read(newfd, rstr, strlen(wstr)); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "read failed: %s\n", strerror(errno)); + goto out; + } + + ret = memcmp(rstr, wstr, strlen(wstr)); + if (ret != 0) { + ret = -1; + fprintf(stderr, "read returning junk\n"); + goto out; + } + + ret = ftruncate(newfd, 0); + if (ret < 0) { + fprintf(stderr, "ftruncate failed : %s\n", strerror(errno)); + goto out; + } + + ret = fstat(newfd, &stbuf); + if (ret < 0) { + fprintf(stderr, "fstat failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchmod(newfd, 0640); + if (ret < 0) { + fprintf(stderr, "fchmod failed : %s\n", strerror(errno)); + goto out; + } + + ret = fchown(newfd, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "fchown failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsync(newfd); + if (ret < 0) { + fprintf(stderr, "fsync failed : %s\n", strerror(errno)); + goto out; + } + + ret = fsetxattr(newfd, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "fsetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fdatasync(newfd); + if (ret < 0) { + fprintf(stderr, "fdatasync failed : %s\n", strerror(errno)); + goto out; + } + + ret = flistxattr(newfd, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "flistxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fgetxattr(newfd, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "fgetxattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = fremovexattr(newfd, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "fremovexattr failed : %s\n", strerror(errno)); + goto out; + } + + ret = 0; out: - if (newfd) - close (newfd); - ret = unlink (filename); - if (ret < 0) - fprintf (stderr, "unlink failed : %s\n", strerror (errno)); + if (newfd) + close(newfd); + ret = unlink(filename); + if (ret < 0) + fprintf(stderr, "unlink failed : %s\n", strerror(errno)); - return ret; + return ret; } int -dir_based_fops (char *dirname) +dir_based_fops(char *dirname) { - int ret = -1; - DIR *dp = NULL; - char buff[255] = {0,}; - struct dirent *dbuff = {0,}; - struct stat stbuff = {0,}; - char newdname[255] = {0,}; - char *cwd = NULL; - - ret = mkdir (dirname, 0755); - if (ret < 0) { - fprintf (stderr, "mkdir failed: %s\n", strerror (errno)); - goto out; - } - - dp = opendir (dirname); - if (dp == NULL) { - fprintf (stderr, "opendir failed: %s\n", strerror (errno)); - goto out; - } - - dbuff = readdir (dp); - if (NULL == dbuff) { - fprintf (stderr, "readdir failed: %s\n", strerror (errno)); - goto out; - } - - ret = closedir (dp); - if (ret < 0) { - fprintf (stderr, "closedir failed: %s\n", strerror (errno)); - goto out; - } - - ret = stat (dirname, &stbuff); - if (ret < 0) { - fprintf (stderr, "stat failed: %s\n", strerror (errno)); - goto out; - } - - ret = chmod (dirname, 0744); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = chown (dirname, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "chmod failed: %s\n", strerror (errno)); - goto out; - } - - ret = setxattr (dirname, "trusted.xattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "setxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = listxattr (dirname, NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "listxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = getxattr (dirname, "trusted.xattr-test", NULL, 0); - if (ret <= 0) { - ret = -1; - fprintf (stderr, "getxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = removexattr (dirname, "trusted.xattr-test"); - if (ret < 0) { - fprintf (stderr, "removexattr failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newdname, dirname); - strcat (newdname, "/../"); - ret = chdir (newdname); - if (ret < 0) { - fprintf (stderr, "chdir failed: %s\n", strerror (errno)); - goto out; - } - - cwd = getcwd (buff, 255); - if (NULL == cwd) { - fprintf (stderr, "getcwd failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newdname, dirname); - strcat (newdname, "new"); - ret = rename (dirname, newdname); - if (ret < 0) { - fprintf (stderr, "rename failed: %s\n", strerror (errno)); - goto out; - } - - ret = rmdir (newdname); - if (ret < 0) { - fprintf (stderr, "rmdir failed: %s\n", strerror (errno)); - return ret; - } + int ret = -1; + DIR *dp = NULL; + char buff[255] = { + 0, + }; + struct dirent *dbuff = { + 0, + }; + struct stat stbuff = { + 0, + }; + char newdname[255] = { + 0, + }; + char *cwd = NULL; + + ret = mkdir(dirname, 0755); + if (ret < 0) { + fprintf(stderr, "mkdir failed: %s\n", strerror(errno)); + goto out; + } + + dp = opendir(dirname); + if (dp == NULL) { + fprintf(stderr, "opendir failed: %s\n", strerror(errno)); + goto out; + } + + dbuff = readdir(dp); + if (NULL == dbuff) { + fprintf(stderr, "readdir failed: %s\n", strerror(errno)); + goto out; + } + + ret = closedir(dp); + if (ret < 0) { + fprintf(stderr, "closedir failed: %s\n", strerror(errno)); + goto out; + } + + ret = stat(dirname, &stbuff); + if (ret < 0) { + fprintf(stderr, "stat failed: %s\n", strerror(errno)); + goto out; + } + + ret = chmod(dirname, 0744); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = chown(dirname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "chmod failed: %s\n", strerror(errno)); + goto out; + } + + ret = setxattr(dirname, "trusted.xattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "setxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = listxattr(dirname, NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "listxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = getxattr(dirname, "trusted.xattr-test", NULL, 0); + if (ret <= 0) { + ret = -1; + fprintf(stderr, "getxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = removexattr(dirname, "trusted.xattr-test"); + if (ret < 0) { + fprintf(stderr, "removexattr failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "/../"); + ret = chdir(newdname); + if (ret < 0) { + fprintf(stderr, "chdir failed: %s\n", strerror(errno)); + goto out; + } + + cwd = getcwd(buff, 255); + if (NULL == cwd) { + fprintf(stderr, "getcwd failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newdname, dirname); + strcat(newdname, "new"); + ret = rename(dirname, newdname); + if (ret < 0) { + fprintf(stderr, "rename failed: %s\n", strerror(errno)); + goto out; + } + + ret = rmdir(newdname); + if (ret < 0) { + fprintf(stderr, "rmdir failed: %s\n", strerror(errno)); + return ret; + } out: - rmdir (dirname); - return ret; + rmdir(dirname); + return ret; } int -link_based_fops (char *filename) +link_based_fops(char *filename) { - int ret = -1; - int fd = 0; - char newname[255] = {0,}; - char linkname[255] = {0,}; - struct stat lstbuf = {0,}; - - fd = creat (filename, 0644); - if (fd < 0) { - fd = 0; - fprintf (stderr, "creat failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (newname, filename); - strcat (newname, "_hlink"); - ret = link (filename, newname); - if (ret < 0) { - fprintf (stderr, "link failed: %s\n", strerror (errno)); - goto out; - } - - ret = unlink (filename); - if (ret < 0) { - fprintf (stderr, "unlink failed: %s\n", strerror (errno)); - goto out; - } - - strcpy (linkname, filename); - strcat (linkname, "_slink"); - ret = symlink (newname, linkname); - if (ret < 0) { - fprintf (stderr, "symlink failed: %s\n", strerror (errno)); - goto out; - } - - ret = lstat (linkname, &lstbuf); - if (ret < 0) { - fprintf (stderr, "lstbuf failed: %s\n", strerror (errno)); - goto out; - } - - ret = lchown (linkname, 10001, 10001); - if (ret < 0) { - fprintf (stderr, "lchown failed: %s\n", strerror (errno)); - goto out; - } - - ret = lsetxattr (linkname, "trusted.lxattr-test", "working", 8, 0); - if (ret < 0) { - fprintf (stderr, "lsetxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = llistxattr (linkname, NULL, 0); - if (ret < 0) { - ret = -1; - fprintf (stderr, "llistxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = lgetxattr (linkname, "trusted.lxattr-test", NULL, 0); - if (ret < 0) { - ret = -1; - fprintf (stderr, "lgetxattr failed: %s\n", strerror (errno)); - goto out; - } - - ret = lremovexattr (linkname, "trusted.lxattr-test"); - if (ret < 0) { - fprintf (stderr, "lremovexattr failed: %s\n", strerror (errno)); - goto out; - } - + int ret = -1; + int fd = 0; + char newname[255] = { + 0, + }; + char linkname[255] = { + 0, + }; + struct stat lstbuf = { + 0, + }; + + fd = creat(filename, 0644); + if (fd < 0) { + fd = 0; + fprintf(stderr, "creat failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(newname, filename); + strcat(newname, "_hlink"); + ret = link(filename, newname); + if (ret < 0) { + fprintf(stderr, "link failed: %s\n", strerror(errno)); + goto out; + } + + ret = unlink(filename); + if (ret < 0) { + fprintf(stderr, "unlink failed: %s\n", strerror(errno)); + goto out; + } + + strcpy(linkname, filename); + strcat(linkname, "_slink"); + ret = symlink(newname, linkname); + if (ret < 0) { + fprintf(stderr, "symlink failed: %s\n", strerror(errno)); + goto out; + } + + ret = lstat(linkname, &lstbuf); + if (ret < 0) { + fprintf(stderr, "lstbuf failed: %s\n", strerror(errno)); + goto out; + } + + ret = lchown(linkname, 10001, 10001); + if (ret < 0) { + fprintf(stderr, "lchown failed: %s\n", strerror(errno)); + goto out; + } + + ret = lsetxattr(linkname, "trusted.lxattr-test", "working", 8, 0); + if (ret < 0) { + fprintf(stderr, "lsetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = llistxattr(linkname, NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "llistxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lgetxattr(linkname, "trusted.lxattr-test", NULL, 0); + if (ret < 0) { + ret = -1; + fprintf(stderr, "lgetxattr failed: %s\n", strerror(errno)); + goto out; + } + + ret = lremovexattr(linkname, "trusted.lxattr-test"); + if (ret < 0) { + fprintf(stderr, "lremovexattr failed: %s\n", strerror(errno)); + goto out; + } out: - if (fd) - close(fd); - unlink (linkname); - unlink (newname); + if (fd) + close(fd); + unlink(linkname); + unlink(newname); } int -test_open_modes (char *filename) +test_open_modes(char *filename) { - int ret = -1; - - ret = generic_open_read_write (filename, O_CREAT|O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_CREAT|O_WRONLY failed: \n"); - goto out; - } - - ret = generic_open_read_write (filename, O_CREAT|O_RDWR); - if (ret != 0) { - fprintf (stderr, "flag O_CREAT|O_RDWR failed\n"); - goto out; - } - - ret = generic_open_read_write (filename, O_CREAT|O_RDONLY); - if (ret != 0) { - fprintf (stderr, "flag O_CREAT|O_RDONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_WRONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_RDWR); - if (0 != ret) { - fprintf (stderr, "flag O_RDWR failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_RDONLY); - if (0 != ret) { - fprintf (stderr, "flag O_RDONLY failed\n"); - goto out; - } - - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_TRUNC|O_WRONLY); - if (3 != ret) { - fprintf (stderr, "flag O_TRUNC|O_WRONLY failed\n"); - goto out; - } + int ret = -1; + + ret = generic_open_read_write(filename, O_CREAT | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_CREAT|O_WRONLY failed: \n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDWR); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDWR failed\n"); + goto out; + } + + ret = generic_open_read_write(filename, O_CREAT | O_RDONLY); + if (ret != 0) { + fprintf(stderr, "flag O_CREAT|O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_WRONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDWR); + if (0 != ret) { + fprintf(stderr, "flag O_RDWR failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_RDONLY); + if (0 != ret) { + fprintf(stderr, "flag O_RDONLY failed\n"); + goto out; + } + + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_TRUNC | O_WRONLY); + if (3 != ret) { + fprintf(stderr, "flag O_TRUNC|O_WRONLY failed\n"); + goto out; + } #if 0 /* undefined behaviour, unable to reliably test */ ret = creat (filename, 0644); @@ -785,84 +833,88 @@ test_open_modes (char *filename) } #endif - ret = generic_open_read_write (filename, O_CREAT|O_RDWR|O_SYNC); - if (0 != ret) { - fprintf (stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n"); - goto out; - } + ret = generic_open_read_write(filename, O_CREAT | O_RDWR | O_SYNC); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_RDWR|O_SYNC failed\n"); + goto out; + } - ret = creat (filename, 0644); - close (ret); - ret = generic_open_read_write (filename, O_CREAT|O_EXCL); - if (0 != ret) { - fprintf (stderr, "flag O_CREAT|O_EXCL failed\n"); - goto out; - } + ret = creat(filename, 0644); + close(ret); + ret = generic_open_read_write(filename, O_CREAT | O_EXCL); + if (0 != ret) { + fprintf(stderr, "flag O_CREAT|O_EXCL failed\n"); + goto out; + } out: - return ret; + return ret; } -int generic_open_read_write (char *filename, int flag) +int +generic_open_read_write(char *filename, int flag) { - int fd = 0; - int ret = -1; - char wstring[50] = {0,}; - char rstring[50] = {0,}; - - fd = open (filename, flag); - if (fd < 0) { - if (flag == O_CREAT|O_EXCL && errno == EEXIST) { - unlink (filename); - return 0; - } - else { - fd = 0; - fprintf (stderr, "open failed: %s\n", strerror (errno)); - return 1; - } - } - - strcpy (wstring, "My string to write\n"); - ret = write (fd, wstring, strlen(wstring)); - if (ret <= 0) { - if (errno != EBADF) { - fprintf (stderr, "write failed: %s\n", strerror (errno)); - close (fd); - unlink(filename); - return 2; - } - } - - ret = lseek (fd, 0, SEEK_SET); - if (ret < 0) { - close (fd); - unlink(filename); - return 4; - } - - ret = read (fd, rstring, strlen(wstring)); - if (ret < 0) { - close (fd); - unlink (filename); - return 3; - } - - /* Compare the rstring with wstring. But we do not want to return - * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or - * O_TRUNC|O_RDONLY. Because in that case we are not writing - * anything to the file.*/ - - ret = memcmp (wstring, rstring, strlen (wstring)); - if (0 != ret && !(flag == O_CREAT|O_RDONLY || flag == O_RDONLY ||\ - flag == O_TRUNC|O_RDONLY)) { - fprintf (stderr, "read is returning junk\n"); - close (fd); - unlink (filename); - return 4; - } - - close (fd); - unlink (filename); - return 0; + int fd = 0; + int ret = -1; + char wstring[50] = { + 0, + }; + char rstring[50] = { + 0, + }; + + fd = open(filename, flag); + if (fd < 0) { + if (flag == O_CREAT | O_EXCL && errno == EEXIST) { + unlink(filename); + return 0; + } else { + fd = 0; + fprintf(stderr, "open failed: %s\n", strerror(errno)); + return 1; + } + } + + strcpy(wstring, "My string to write\n"); + ret = write(fd, wstring, strlen(wstring)); + if (ret <= 0) { + if (errno != EBADF) { + fprintf(stderr, "write failed: %s\n", strerror(errno)); + close(fd); + unlink(filename); + return 2; + } + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + close(fd); + unlink(filename); + return 4; + } + + ret = read(fd, rstring, strlen(wstring)); + if (ret < 0) { + close(fd); + unlink(filename); + return 3; + } + + /* Compare the rstring with wstring. But we do not want to return + * error when the flag is either O_RDONLY, O_CREAT|O_RDONLY or + * O_TRUNC|O_RDONLY. Because in that case we are not writing + * anything to the file.*/ + + ret = memcmp(wstring, rstring, strlen(wstring)); + if (0 != ret && !(flag == O_CREAT | O_RDONLY || flag == O_RDONLY || + flag == O_TRUNC | O_RDONLY)) { + fprintf(stderr, "read is returning junk\n"); + close(fd); + unlink(filename); + return 4; + } + + close(fd); + unlink(filename); + return 0; } diff --git a/extras/thin-arbiter/setup-thin-arbiter.sh b/extras/thin-arbiter/setup-thin-arbiter.sh new file mode 100755 index 00000000000..0681b30ef3f --- /dev/null +++ b/extras/thin-arbiter/setup-thin-arbiter.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Copyright (c) 2018-2019 Red Hat, Inc. <http://www.redhat.com> +# This file is part of GlusterFS. +# +# This file is licensed to you under your choice of the GNU Lesser +# General Public License, version 3 or any later version (LGPLv3 or +# later), or the GNU General Public License, version 2 (GPLv2), in all +# cases as published by the Free Software Foundation. + + +# This tool has been developed to setup thin-arbiter process on a node. +# Seting up a thin arbiter process involves following files - +# 1 - thin-arbiter.vol +# Thin-arbiter (TA) process will use the graph in this file to load the +# required translators. +# 2 - gluster-ta-volume.service (generated by gluster-ta-volume.service.in) +# TA process would be running as systemd service. +# +# TA process uses a location to save TA id files for every subvolume. +# This location can be taken as input from user. Once provided and the +# TA process is started on a node, it can not be changed using this +# script or by any other mean. The same location should be used in +# the gluster CLI when creating thin-arbiter volumes. + +MYPATH=`dirname $0` + +volloc="/var/lib/glusterd/thin-arbiter" +mkdir -p $volloc + +if [ -f /etc/glusterfs/thin-arbiter.vol ]; then + volfile=/etc/glusterfs/thin-arbiter.vol +else + volfile=$MYPATH/thin-arbiter.vol +fi + +tafile="$volloc/thin-arbiter.vol" + + +help () { + echo " " + echo ' This tool helps to setup thin-arbiter (TA) process on a node. + TA process uses a location to save TA id files for every subvolume. + This location can be taken as input from user. Once provided and the + TA process is started on a node, it can not be changed using this script + or by any other mean. The same location should be used in gluster CLI + when creating thin-arbiter volumes. + + usage: setup-thin-arbiter.sh [-s] [-h] + options: + -s - Setup thin-arbiter file path and start process + -h - Show this help message and exit +' +} + +volfile_set_brick_path () { + while read -r line + do + dir=`echo "$line" | cut -d' ' -f 2` + if [ "$dir" = "directory" ]; then + bpath=`echo "$line" | cut -d' ' -f 3` + sed -i -- 's?'$bpath'?'$1'?g' $tafile + return + fi + done < $tafile +} + +check_ta_proc () { + pro=`ps aux | grep thin-arbiter.vol | grep "volfile-id"` + if [ "${pro}" = '' ]; then + echo "" + else + curr_loc=`cat $volloc/thin-arbiter.vol | grep option | grep directory` + loc=`echo "${curr_loc##* }"` + echo "******************************************************" + echo "Error:" + echo "Thin-arbiter process is running with thin-arbiter path = $loc" + echo "Can not change TA path on this host now." + echo "$pro" + echo "******************************************************" + exit 1 + fi +} + +getpath () { + check_ta_proc + echo "******************************************************" + echo "User will be required to enter a path/folder for arbiter volume." + echo "Please note that this path will be used for ALL VOLUMES using this" + echo "node to host thin-arbiter. After setting, if a volume" + echo "has been created using this host and path then path for" + echo "thin-arbiter can not be changed " + echo "******************************************************" + echo " " + while true; + do + echo -n "Enter brick path for thin arbiter volumes: " + echo " " + read tapath + if [ "${tapath}" = '' ]; then + echo "Please enter valid path" + continue + else + echo "Entered brick path : $tapath " + echo "Please note that this brick path will be used for ALL" + echo "VOLUMES using this node to host thin-arbiter brick" + echo -n "Want to continue? (y/N): " + echo " " + read cont + + if [ "${cont}" = 'N' ] || [ "${cont}" = 'n' ]; then + exit 0 + else + break + fi + fi + done +} + +setup () { + getpath + mkdir -p $tapath/.glusterfs/indices + if [ -d $tapath/.glusterfs/indices ]; then + echo " " + else + echo "Could not create $tapath/.glusterfs/indices directory, check provided ta path." + exit 1 + fi + + cp -f --backup --suffix=_old $volfile $volloc/thin-arbiter.vol + volfile_set_brick_path "$tapath" + + echo "Directory path to be used for thin-arbiter volume is: $tapath" + echo " " + echo "========================================================" + + if [ -f /usr/lib/systemd/system/gluster-ta-volume.service ]; then + echo "Starting thin-arbiter process" + else + cp $MYPATH/../systemd/gluster-ta-volume.service /etc/systemd/system/ + echo "Starting thin-arbiter process" + chmod 0644 /etc/systemd/system/gluster-ta-volume.service + fi + + systemctl daemon-reload + systemctl enable gluster-ta-volume + systemctl stop gluster-ta-volume + systemctl start gluster-ta-volume + + if [ $? == 0 ]; then + echo "thin-arbiter process has been setup and running" + else + echo "Failed to setup thin arbiter" + exit 1 + fi + +} + +main() +{ + + if [ "$#" -ne 1 ]; then + help + exit 0 + fi + + while getopts "sh" opt; do + case $opt in + h) + help + exit 0 + ;; + s) + setup + exit 0 + ;; + *) + help + exit 0 + ;; + esac + done +} + +main "$@" diff --git a/extras/thin-arbiter/thin-arbiter.vol b/extras/thin-arbiter/thin-arbiter.vol new file mode 100644 index 00000000000..c76babc7b3c --- /dev/null +++ b/extras/thin-arbiter/thin-arbiter.vol @@ -0,0 +1,57 @@ +volume ta-posix + type storage/posix + option directory /mnt/thin-arbiter +end-volume + +volume ta-thin-arbiter + type features/thin-arbiter + subvolumes ta-posix +end-volume + +volume ta-locks + type features/locks + option notify-contention yes + subvolumes ta-thin-arbiter +end-volume + +volume ta-upcall + type features/upcall + option cache-invalidation off + subvolumes ta-locks +end-volume + +volume ta-io-threads + type performance/io-threads + subvolumes ta-upcall +end-volume + +volume ta-index + type features/index + option xattrop-pending-watchlist trusted.afr.ta- + option xattrop-dirty-watchlist trusted.afr.dirty + option index-base /mnt/thin-arbiter/.glusterfs/indices + subvolumes ta-io-threads +end-volume + +volume /mnt/thin-arbiter + type debug/io-stats + option count-fop-hits off + option latency-measurement off + option unique-id /mnt/thin-arbiter + subvolumes ta-index +end-volume + +volume ta-server + type protocol/server + option transport.listen-backlog 10 + option transport.socket.keepalive-count 9 + option transport.socket.keepalive-interval 2 + option transport.socket.keepalive-time 20 + option transport.tcp-user-timeout 0 + option transport.socket.keepalive 1 + option auth.addr./mnt/thin-arbiter.allow * + option auth-path /mnt/thin-arbiter + option transport.address-family inet + option transport-type tcp + subvolumes /mnt/thin-arbiter +end-volume diff --git a/extras/volfilter.py b/extras/volfilter.py index 0ca456a7882..5558a1beff4 100644 --- a/extras/volfilter.py +++ b/extras/volfilter.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License * along # with HekaFS. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function import copy import string import sys @@ -35,7 +36,7 @@ good_xlators = [ "storage/posix", ] -def copy_stack (old_xl,suffix,recursive=False): +def copy_stack (old_xl, suffix, recursive=False): if recursive: new_name = old_xl.name + "-" + suffix else: @@ -45,7 +46,7 @@ def copy_stack (old_xl,suffix,recursive=False): # The results with normal assignment here are . . . amusing. new_xl.opts = copy.deepcopy(old_xl.opts) for sv in old_xl.subvols: - new_xl.subvols.append(copy_stack(sv,suffix,True)) + new_xl.subvols.append(copy_stack(sv, suffix, True)) # Patch up the path at the bottom. if new_xl.type == "storage/posix": new_xl.opts["directory"] += ("/" + suffix) @@ -63,10 +64,10 @@ def cleanup (parent, graph): parent.opts["transport-type"] = "ssl" sv = [] for child in parent.subvols: - sv.append(cleanup(child,graph)) + sv.append(cleanup(child, graph)) parent.subvols = sv else: - parent = cleanup(parent.subvols[0],graph) + parent = cleanup(parent.subvols[0], graph) return parent class Translator: @@ -82,8 +83,8 @@ class Translator: def load (path): # If it's a string, open it; otherwise, assume it's already a # file-like object (most notably from urllib*). - if type(path) in types.StringTypes: - fp = file(path,"r") + if type(path) in (str,): + fp = file(path, "r") else: fp = path all_xlators = {} @@ -98,16 +99,16 @@ def load (path): continue if text[0] == "volume": if xlator: - raise RuntimeError, "nested volume definition" + raise RuntimeError("nested volume definition") xlator = Translator(text[1]) continue if not xlator: - raise RuntimeError, "text outside volume definition" + raise RuntimeError("text outside volume definition") if text[0] == "type": xlator.type = text[1] continue if text[0] == "option": - xlator.opts[text[1]] = string.join(text[2:]) + xlator.opts[text[1]] = ''.join(text[2:]) continue if text[0] == "subvolumes": for sv in text[1:]: @@ -118,25 +119,25 @@ def load (path): last_xlator = xlator xlator = None continue - raise RuntimeError, "unrecognized keyword %s" % text[0] + raise RuntimeError("unrecognized keyword %s" % text[0]) if xlator: - raise RuntimeError, "unclosed volume definition" + raise RuntimeError("unclosed volume definition") return all_xlators, last_xlator def generate (graph, last, stream=sys.stdout): for sv in last.subvols: if not sv.dumped: - generate(graph,sv,stream) - print >> stream, "" + generate(graph, sv, stream) + print("", file=stream) sv.dumped = True - print >> stream, "volume %s" % last.name - print >> stream, " type %s" % last.type - for k, v in last.opts.iteritems(): - print >> stream, " option %s %s" % (k, v) + print("volume %s" % last.name, file=stream) + print(" type %s" % last.type, file=stream) + for k, v in last.opts.items(): + print(" option %s %s" % (k, v), file=stream) if last.subvols: - print >> stream, " subvolumes %s" % string.join( - [ sv.name for sv in last.subvols ]) - print >> stream, "end-volume" + print(" subvolumes %s" % ''.join( + [ sv.name for sv in last.subvols ]), file=stream) + print("end-volume", file=stream) def push_filter (graph, old_xl, filt_type, opts={}): suffix = "-" + old_xl.type.split("/")[1] @@ -156,7 +157,7 @@ def push_filter (graph, old_xl, filt_type, opts={}): def delete (graph, victim): if len(victim.subvols) != 1: - raise RuntimeError, "attempt to delete non-unary translator" + raise RuntimeError("attempt to delete non-unary translator") for xl in graph.itervalues(): while xl.subvols.count(victim): i = xl.subvols.index(victim) @@ -164,4 +165,4 @@ def delete (graph, victim): if __name__ == "__main__": graph, last = load(sys.argv[1]) - generate(graph,last) + generate(graph, last) diff --git a/extras/who-wrote-glusterfs/gitdm.aliases b/extras/who-wrote-glusterfs/gitdm.aliases index 1fcdda7138b..901c12418e3 100644 --- a/extras/who-wrote-glusterfs/gitdm.aliases +++ b/extras/who-wrote-glusterfs/gitdm.aliases @@ -55,3 +55,4 @@ vijay@dev.gluster.com vbellur@redhat.com vijaykumar.koppad@gmail.com vkoppad@redhat.com vikas@zresearch.com vikas@gluster.com shishirng@gluster.com sgowda@redhat.com +potatogim@potatogim.net potatogim@gluesys.com diff --git a/extras/who-wrote-glusterfs/gitdm.domain-map b/extras/who-wrote-glusterfs/gitdm.domain-map index d9ea7f235a0..7cd2bbd605b 100644 --- a/extras/who-wrote-glusterfs/gitdm.domain-map +++ b/extras/who-wrote-glusterfs/gitdm.domain-map @@ -4,6 +4,7 @@ active.by ActiveCloud appeartv.com Appear TV cern.ch CERN +cmss.chinamobile.com China Mobile(Suzhou) Software Technology datalab.es DataLab S.L. fb.com Facebook fedoraproject.org Fedora Project @@ -25,3 +26,4 @@ stepping-stone.ch stepping stone GmbH xtaotech.com XTAO Co. yahoo.in (personal contributions) zresearch.com Red Hat +gluesys.com Gluesys |