From b47528cc4bb7a308c74d4dae7c2adcf9f7392f4e Mon Sep 17 00:00:00 2001 From: Lakshmipathi Date: Fri, 8 Jul 2011 11:51:53 +0530 Subject: Added community provided scripts for lock and rename --- INFO | 14 + README | 6 + community-scripts/locks/lock.php | 22 ++ community-scripts/locks/locky.sh | 10 + community-scripts/rename/atomic/bug1034.tar | Bin 0 -> 20480 bytes community-scripts/rename/atomic/bug1034/README | 88 +++++ community-scripts/rename/atomic/bug1034/gt.h | 12 + community-scripts/rename/atomic/bug1034/gt_init.c | 114 +++++++ community-scripts/rename/atomic/bug1034/gt_read.c | 131 +++++++ .../rename/atomic/bug1034/gt_rename.c | 86 +++++ .../rename/atomic/bug1034/volume.info | 10 + community-scripts/rename/atomic/reader.c | 23 ++ community-scripts/rename/atomic/writer.c | 48 +++ community-scripts/rename/index.c | 333 ++++++++++++++++++ community-scripts/rename/rename.sh | 379 +++++++++++++++++++++ community-scripts/rename/repo.py | 103 ++++++ 16 files changed, 1379 insertions(+) create mode 100644 INFO create mode 100644 community-scripts/locks/lock.php create mode 100644 community-scripts/locks/locky.sh create mode 100644 community-scripts/rename/atomic/bug1034.tar create mode 100644 community-scripts/rename/atomic/bug1034/README create mode 100644 community-scripts/rename/atomic/bug1034/gt.h create mode 100644 community-scripts/rename/atomic/bug1034/gt_init.c create mode 100644 community-scripts/rename/atomic/bug1034/gt_read.c create mode 100644 community-scripts/rename/atomic/bug1034/gt_rename.c create mode 100644 community-scripts/rename/atomic/bug1034/volume.info create mode 100644 community-scripts/rename/atomic/reader.c create mode 100644 community-scripts/rename/atomic/writer.c create mode 100644 community-scripts/rename/index.c create mode 100644 community-scripts/rename/rename.sh create mode 100644 community-scripts/rename/repo.py diff --git a/INFO b/INFO new file mode 100644 index 0000000..5fe20aa --- /dev/null +++ b/INFO @@ -0,0 +1,14 @@ +sanity/nightly_valgrind/ ---> script used for nightly valgrind run. +sanity/dev_sanity/ ---> script used for dev_sanity test. +=============================================== +community-scripts/rename/ ---> few scripts which catch rename related issues in dht(note it might expose rename issues in other xlators too). + +a)repo.py ---> create 5 directories 'input{1,2,3,4,5}' with each directory has 1500 files and create 'testdir'. Run the program +b)rename.sh ---> rename script +c)index.c ---> dovecot program + +community-scripts/rename/atomic ---> Bug 1034 - rename() is not atomic +=============================================== +community-scripts/locks/locky.sh --->Configuration on the command line and (serious) locking problems +community-scripts/locks/lock.php ---> Bug 1017 - Locking deadlock when upgrading lock +=============================================== diff --git a/README b/README index e69de29..9be9ae9 100644 --- a/README +++ b/README @@ -0,0 +1,6 @@ + README + ------ +Add sanity scripts under directory "sanity" +Add community provided scripts under "community-scripts" + +Read INFO file for script details. diff --git a/community-scripts/locks/lock.php b/community-scripts/locks/lock.php new file mode 100644 index 0000000..dc4d039 --- /dev/null +++ b/community-scripts/locks/lock.php @@ -0,0 +1,22 @@ +#/usr/bin/php + diff --git a/community-scripts/locks/locky.sh b/community-scripts/locks/locky.sh new file mode 100644 index 0000000..8d9518b --- /dev/null +++ b/community-scripts/locks/locky.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +for((i = 0; i < 200; ++i)); do + flock counter -c ' + read value < counter + echo $value + value=$(( value + 1 )) + echo ${value} > counter + ' + done diff --git a/community-scripts/rename/atomic/bug1034.tar b/community-scripts/rename/atomic/bug1034.tar new file mode 100644 index 0000000..88b5eb5 Binary files /dev/null and b/community-scripts/rename/atomic/bug1034.tar differ diff --git a/community-scripts/rename/atomic/bug1034/README b/community-scripts/rename/atomic/bug1034/README new file mode 100644 index 0000000..fd0acd7 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/README @@ -0,0 +1,88 @@ +*** Cluster and Gluster Configuration + +The cluster has MAX nodes. Each node has a gluster brick. +The bricks are assembled into a single gluster volume, using +pure replication. The gluster volume is mounted at the same +BASE_PATH on each node. + +Edit BASE_PATH in gt.h to the appropriate path before +compiling. + +Compile the test programs + cc gt_init.c -o gt_init + cc gt_rename.c -o gt_rename + cc gt_read.c -o gt_read +And distribute them to all nodes of the cluster. + +*** Test Configuration + +There are three test programs. + +One instance of gt_init is run once, on any node in +the cluster, before the other test processes are started. +This cleans up and initializes the contents of BASE_PATH, +then terminates. + +An instance of gt_rename runs on every node. Each instance is +started with a numeral from 1 to MAX as a command line argument. +These continuously replace a set of files by renaming a temp +file on top of existing files. + +An instance of gt_read runs on every node. Each instance is +started with a numeral from 1 to MAX as a command line argument. +These continuously read all the files touched by the set of +gt_rename processes. + +I.e., with cluster nodes numbered 1 to MAX, on node n start: + gt_rename n & + gt_read MAX & + +*** File Structure + +BASE_PATH/1/WA_RC_0 + /WA_RC_1 + /WA_RC_2 + /... + /MAX/WA_RC_0 + /WA_RC_1 + /WA_RC_2 + +There is a directory under BASE_PATH for each node, +numbered 1 through MAX. + +gt_rename processes manipulate files only within +their node's directory. + +gt_read test process reads all files and writes no +file. + +Every file is written by only one test process. +The contents of every directory (not directory tree) +is written by only one test process. + +Every file is read by MAX test processes other than +the single writing test process. + +*** Test Operation + +The gt_init process destroys any existing file structure +under BASE_PATH, then creates and initializes the contents +of the file structure. + +The gt_rename processes always write a fixed distinctive +value into each file they write. They open a temp file, write a +fixed value, and rename the file onto the standard file. +The fixed value is different for every standard file. + +The gt_rename and gt_read processes always test each file for +existence, then check the contents for the expected distinctive +value, for each file they access. Apparently missing files or +unexpected contents are logged. + +The gt_rename and gt_read processes iterate every CYCLE_TIME ++[0..CYCLE_JITTER) milliseconds until terminated. + +By observing the output of the gt_read processes, the +characteristics of bug1034 can be seen. On node n, gt_read +gets ENOENT for open and for read-after-successful-open on +BASE_PATH/n/*. diff --git a/community-scripts/rename/atomic/bug1034/gt.h b/community-scripts/rename/atomic/bug1034/gt.h new file mode 100644 index 0000000..da6a57c --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt.h @@ -0,0 +1,12 @@ +#ifndef _BUG1034_TEST_ +#define _BUG1034_TEST_ + +// #define BASE_PATH "/common_mount_point/test_directory" +#define BASE_PATH "/sme_cluster/global/bug1034_test" + +#define CYCLE_TIME 5 +#define CYCLE_JITTER 5 + +#define SUBDIR_FILES 3 + +#endif /* ! _BUG1034_TEST_ */ diff --git a/community-scripts/rename/atomic/bug1034/gt_init.c b/community-scripts/rename/atomic/bug1034/gt_init.c new file mode 100644 index 0000000..f3dadf0 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_init.c @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node_count\n", progname); + exit(1); +} + +// rm -rf +static void +delete_tree(char *rootpath) +{ + int retval; + struct stat statbuf; + + retval = lstat(rootpath, &statbuf); + if (retval < 0) { + return; + } + + if (S_ISDIR(statbuf.st_mode)) { + DIR *dir; + struct dirent *dent; + char pathbuf[PATH_MAX]; + + dir = opendir(rootpath); + while ((dent = readdir(dir)) != NULL) { + if (0 == strcmp(".", dent->d_name)) { + continue; + } + if (0 == strcmp("..", dent->d_name)) { + continue; + } + snprintf(pathbuf, sizeof(pathbuf), + "%s/%s", rootpath, dent->d_name); + delete_tree(pathbuf); + } + closedir(dir); + + (void)rmdir(rootpath); + } else { + (void)unlink(rootpath); + } +} + +static void +create_subdir(char *dirpath) +{ + int i; + + mkdir(dirpath, 0775); + + for (i = 0; i < SUBDIR_FILES; i++) { + char pathbuf[PATH_MAX]; + int fd; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/WA_RC_%d", dirpath, i); + fd = open(pathbuf, O_RDWR|O_CREAT, 0664); + write(fd, pathbuf, strlen(pathbuf)); + close(fd); + } +} + +static void +create_tree(char *rootpath, int node_count) +{ + int n; + + mkdir(rootpath, 0775); + + for (n = 1; n <= node_count; n++) { + char pathbuf[PATH_MAX]; + int fd; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d", rootpath, n); + mkdir(pathbuf, 0775); + create_subdir(pathbuf); + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node_count; + + if (argc != 2) { + usage(argv[0]); + } + + node_count = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + delete_tree(BASE_PATH); + + create_tree(BASE_PATH, node_count); + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/gt_read.c b/community-scripts/rename/atomic/bug1034/gt_read.c new file mode 100644 index 0000000..7d07d59 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_read.c @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node_count\n", progname); + exit(1); +} + +static void +dump_data(char *buf, size_t buflen) +{ + int i; + + fprintf(stderr, "raw data: "); + for (i = 0; i < buflen; i++) { + int c = buf[i]; + + if (isgraph(c)) { + fprintf(stderr, "%c", c); + } else if (isspace(c)) { + switch (c) { + case ' ': + fprintf(stderr, " "); + break; + case '\f': + fprintf(stderr, "\f"); + break; + case '\n': + fprintf(stderr, "\n"); + break; + case '\r': + fprintf(stderr, "\r"); + break; + case '\t': + fprintf(stderr, "\t"); + break; + case '\v': + fprintf(stderr, "\v"); + break; + default: + fprintf(stderr, "\%03o", c); + break; + } + } else { + fprintf(stderr, "\%03o", c); + } + } + fprintf(stderr, "\n"); +} + +static void +touch_tree(char *rootpath, int node_count) +{ + char pathbuf[PATH_MAX]; + char databuf[PATH_MAX]; + int fd; + int node; + int i; + int retval; + + for (node = 1; node <= node_count; node++) { + for (i = 0; i < SUBDIR_FILES; i++) { + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d/WA_RC_%d", rootpath, node, i); + fd = open(pathbuf, O_RDONLY); + if (0 > fd) { + fprintf(stderr, "gt_read: %s: open %s\n", + strerror(errno), pathbuf); + continue; + } + retval = read(fd, databuf, sizeof(databuf)); + close(fd); + if (0 > retval) { + fprintf(stderr, "gt_read: %s: read %s\n", + strerror(errno), pathbuf); + } else if (retval != strlen(pathbuf)) { + fprintf(stderr, + "gt_read: incomplete read: read %s\n", + pathbuf); + } else { + databuf[retval] = '\0'; + if (0 != strcmp(pathbuf, databuf)) { + fprintf(stderr, + "gt_read: bad data: read %s\n", + pathbuf); + dump_data(databuf, strlen(pathbuf)); + } + } + } + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node_count; + + if (argc != 2) { + usage(argv[0]); + } + + node_count = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + while (1) { + int snooze; + + touch_tree(BASE_PATH, node_count); + + snooze = CYCLE_TIME + random()%CYCLE_JITTER; + usleep(1000*snooze); + } + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/gt_rename.c b/community-scripts/rename/atomic/bug1034/gt_rename.c new file mode 100644 index 0000000..96ecce0 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_rename.c @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node\n", progname); + exit(1); +} + +static void +touch_tree(char *rootpath, int node) +{ + char pathbuf[PATH_MAX]; + char databuf[PATH_MAX]; + int fd; + int i; + int retval; + + for (i = 0; i < SUBDIR_FILES; i++) { + char tempbuf[PATH_MAX]; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d/WA_RC_%d", rootpath, node, i); + snprintf(tempbuf, sizeof(tempbuf), + "%s/%d/WA_RC_%d.temp", rootpath, node, i); + fd = open(tempbuf, O_RDWR|O_CREAT, 0644); + if (0 > fd) { + fprintf(stderr, "gt_rename: %s: open %s\n", + strerror(errno), tempbuf); + continue; + } + retval = write(fd, pathbuf, strlen(pathbuf)); + if (0 > retval) { + fprintf(stderr, "gt_rename: %s: write %s\n", + strerror(errno), pathbuf); + } else if (retval != strlen(pathbuf)) { + fprintf(stderr, "gt_rename: incomplete write: write %s\n", + pathbuf); + } + close(fd); + + retval = rename(tempbuf, pathbuf); + if (0 > retval) { + fprintf(stderr, "gt_rename: %s rename %s\n", + strerror(errno), pathbuf); + } + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node; + + if (argc != 2) { + usage(argv[0]); + } + + node = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + while (1) { + int snooze; + + touch_tree(BASE_PATH, node); + + snooze = CYCLE_TIME + random()%CYCLE_JITTER; + usleep(1000*snooze); + } + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/volume.info b/community-scripts/rename/atomic/bug1034/volume.info new file mode 100644 index 0000000..68d654e --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/volume.info @@ -0,0 +1,10 @@ +type=2 +count=2 +status=1 +sub_count=2 +version=1 +transport-type=0 +volume-id=01234567-89ab-cdef-0123-456789abcdef +brick-0=host0:-our-brick-location +brick-1=host1:-our-brick-location + diff --git a/community-scripts/rename/atomic/reader.c b/community-scripts/rename/atomic/reader.c new file mode 100644 index 0000000..d25d9d8 --- /dev/null +++ b/community-scripts/rename/atomic/reader.c @@ -0,0 +1,23 @@ +/* + gcc reader.c -o reader -Wall +*/ +#include +#include +#include +#include + +int main(void) +{ + int fd; + + for (;;) { + usleep(rand() % 1000); + fd = open("dovecot.index", O_RDONLY); + if (fd == -1) { + perror("open(dovecot.index)"); + break; + } + close(fd); + } + return 0; +} diff --git a/community-scripts/rename/atomic/writer.c b/community-scripts/rename/atomic/writer.c new file mode 100644 index 0000000..16d07e1 --- /dev/null +++ b/community-scripts/rename/atomic/writer.c @@ -0,0 +1,48 @@ +/* + gcc writer.c -o writer -Wall +*/ +#include +#include +#include +#include + +int main(void) +{ + int fd_old, fd_new, ret; + + for (;;) { + fd_old = open("dovecot.index", O_RDWR); + if (fd_old == -1) { + perror("open(dovecot.index"); + break; + } + + usleep(rand() % 1000); + fd_new = creat("dovecot.index.tmp", 0600); + if (fd_new == -1) { + perror("creat(dovecot.index.tmp)"); + break; + } + write(fd_new, "foo", 3); + close(fd_new); + + ret = link("dovecot.index", "dovecot.index.backup.tmp"); + if (ret < 0) { + perror("link(dovecot.index, dovecot.index.backup.tmp)"); + break; + } + if (rename("dovecot.index.backup.tmp", "dovecot.index.backup") < 0) { + perror("rename(dovecot.index.backup.tmp, dovecot.index.backup)"); + break; + } + + usleep(rand() % 1000); + if (rename("dovecot.index.tmp", "dovecot.index") < 0) { + perror("rename(dovecot.index.tmp, dovecot.index)"); + break; + } + usleep(rand() % 1000); + close(fd_old); + } + return 0; +} diff --git a/community-scripts/rename/index.c b/community-scripts/rename/index.c new file mode 100644 index 0000000..4382095 --- /dev/null +++ b/community-scripts/rename/index.c @@ -0,0 +1,333 @@ +/* + gcc index.c -o index -Wall -g + ./index & + ./index & + ./index & + + abort()s on failure, runs forever on non-failure +*/ +#define _XOPEN_SOURCE 500 /* for pread() */ +#define _BSD_SOURCE /* for major(), minor() */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct mail_index { + const char *filepath; + int fd; + int log_fd; +}; + +struct mail_index_header { + /* major version is increased only when you can't have backwards + compatibility. minor version is increased when header size is + increased to contain new non-critical fields. */ + uint8_t major_version; + uint8_t minor_version; + + uint16_t base_header_size; + uint32_t header_size; /* base + extended header size */ + uint32_t record_size; + + uint8_t compat_flags; /* enum mail_index_header_compat_flags */ + uint8_t unused[3]; + + uint32_t indexid; + uint32_t flags; + + uint32_t uid_validity; + uint32_t next_uid; + + uint32_t messages_count; + uint32_t unused_old_recent_messages_count; + uint32_t seen_messages_count; + uint32_t deleted_messages_count; + + uint32_t first_recent_uid; + /* these UIDs may not exist and may not even be unseen/deleted */ + uint32_t first_unseen_uid_lowwater; + uint32_t first_deleted_uid_lowwater; + + uint32_t log_file_seq; + /* non-external records between tail..head haven't been committed to + mailbox yet. */ + uint32_t log_file_tail_offset; + uint32_t log_file_head_offset; + + uint64_t sync_size; + uint32_t sync_stamp; + + /* daily first UIDs that have been added to index. */ + uint32_t day_stamp; + uint32_t day_first_uid[8]; +}; + +struct mail_index_record { + uint32_t uid; + uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */ +}; +int mail_index_try_open_only(struct mail_index *index) +{ + index->fd = open(index->filepath, O_RDWR); + if (index->fd == -1) { + if (errno != ENOENT) + abort(); + + /* have to create it */ + return 0; + } + return 1; +} + +void mail_index_close_file(struct mail_index *index) +{ + if (index->fd != -1) { + if (close(index->fd) < 0) + abort(); + index->fd = -1; + } +} + +int mail_index_reopen_if_changed(struct mail_index *index) +{ + struct stat st1, st2; + + if (index->fd == -1) + return mail_index_try_open_only(index); + + //nfs_flush_file_handle_cache(index->filepath); + if (stat(index->filepath, &st2) < 0) + abort(); + +#define CMP_DEV_T(a, b) (major(a) == major(b) && minor(a) == minor(b)) + if (fstat(index->fd, &st1) < 0) { + abort(); + } else if (st1.st_ino == st2.st_ino && + CMP_DEV_T(st1.st_dev, st2.st_dev)) { + /* the same file */ + return 1; + } + + /* new file */ + mail_index_close_file(index); + return mail_index_try_open_only(index); +} + +static int mail_index_read_header(struct mail_index *index, + void *buf, size_t buf_size, size_t *pos_r) +{ + size_t pos; + int ret; + + memset(buf, 0, sizeof(struct mail_index_header)); + + pos = 0; + do { + ret = pread(index->fd, (char*)buf + pos, + buf_size - pos, pos); + if (ret <= 0) + abort(); + if (ret > 0) + pos += ret; + } while (ret > 0 && pos < sizeof(struct mail_index_header)); + + *pos_r = pos; + return ret; +} + +static int +mail_index_read_map(struct mail_index *index, off_t file_size) +{ + const struct mail_index_header *hdr; + unsigned char read_buf[8192]; + const void *buf; + ssize_t ret; + size_t pos, records_size; + unsigned int records_count = 0, extra; + + ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos); + buf = read_buf; hdr = buf; + + if (ret > 0) { + /* header read, read the records now. */ + records_size = (size_t)hdr->messages_count * hdr->record_size; + records_count = hdr->messages_count; + + if (file_size - hdr->header_size < records_size || + (hdr->record_size != 0 && + records_size / hdr->record_size != hdr->messages_count)) + abort(); + + if (pos <= hdr->header_size) + extra = 0; + else + extra = pos - hdr->header_size; + if (records_size > extra) { + void *data = malloc(records_size - extra); + ret = pread(index->fd, data, records_size - extra, + hdr->header_size + extra); + if (ret != records_size - extra) abort(); + free(data); + } + } + return 1; +} + +static int file_lock_do(int fd, int lock_type, int timeout_secs) +{ + struct flock fl; + int ret; + + fl.l_type = lock_type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + + if (timeout_secs != 0) + alarm(timeout_secs); + ret = fcntl(fd, timeout_secs != 0 ? F_SETLKW : F_SETLK, &fl); + if (timeout_secs != 0) + alarm(0); + + if (ret == 0) + return 1; + + if (timeout_secs == 0 && + (errno == EACCES || errno == EAGAIN)) { + /* locked by another process */ + return 0; + } + abort(); +} + +static int mail_index_map_latest_file(struct mail_index *index) +{ + struct stat st; + off_t file_size; + int ret; + + ret = mail_index_reopen_if_changed(index); + if (ret <= 0) { + if (ret < 0) + return -1; + + /* the index file is lost/broken. */ + return 1; + } + + if (file_lock_do(index->fd, F_RDLCK, 120) == 0) abort(); + + //nfs_flush_attr_cache_fd_locked(index->filepath, index->fd); + if (fstat(index->fd, &st) < 0) + abort(); + file_size = st.st_size; + + ret = mail_index_read_map(index, file_size); + if (file_lock_do(index->fd, F_UNLCK, 0) == 0) abort(); + return 1; +} + +static int mail_index_create_backup(struct mail_index *index) +{ + char backup_path[1024], tmp_backup_path[1024]; + int ret; + + snprintf(backup_path, sizeof(backup_path), "%s.backup", index->filepath); + snprintf(tmp_backup_path, sizeof(tmp_backup_path), "%s.tmp", backup_path); + ret = link(index->filepath, tmp_backup_path); + if (ret < 0 && errno == EEXIST) { + if (unlink(tmp_backup_path) < 0 && errno != ENOENT) + abort(); + ret = link(index->filepath, tmp_backup_path); + } + if (ret < 0) { + if (errno == ENOENT) { + /* no dovecot.index file, ignore */ + return 0; + } + abort(); + } + + if (rename(tmp_backup_path, backup_path) < 0) + abort(); + return 0; +} + +static void mail_index_recreate(struct mail_index *index) +{ + struct mail_index_header hdr; + struct mail_index_record *recs; + char path[1024]; + int fd; + unsigned int i, size; + + snprintf(path, sizeof(path), "%s.tmp", index->filepath); + fd = open(path, O_RDWR|O_CREAT|O_TRUNC, 0600); + if (fd == -1) + abort(); + + memset(&hdr, 0, sizeof(hdr)); + hdr.base_header_size = hdr.header_size = sizeof(hdr); + hdr.record_size = sizeof(*recs); + hdr.messages_count = (rand() % 10000) * 100; + + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) abort(); + size = sizeof(*recs) * hdr.messages_count; + recs = calloc(size, 1); + for (i = 0; i < hdr.messages_count; i++) + recs[i].uid = i + 1; + if (write(fd, recs, size) != size) abort(); + free(recs); + if (fdatasync(fd) < 0) + abort(); + if (close(fd) < 0) + abort(); + mail_index_create_backup(index); + + if (rename(path, index->filepath) < 0) { + perror("rename()"); + abort(); + } +} + +static int log_lock(struct mail_index *index) +{ + if (index->log_fd == -1) { + index->log_fd = open("/mnt/gluster/dovecot.index.log", O_CREAT | O_RDWR, 0600); + if (index->log_fd == -1) + abort(); + } + return file_lock_do(index->log_fd, F_WRLCK, 0); +} + +static void log_unlock(struct mail_index *index) +{ + if (file_lock_do(index->log_fd, F_UNLCK, 0) == 0) abort(); +} + +int main(void) +{ + struct mail_index index; + + memset(&index, 0, sizeof(index)); + index.fd = -1; + index.log_fd = -1; + index.filepath = "/mnt/gluster/dovecot.index"; + + for (;;) { + if (rand() % 100 < 70) + mail_index_map_latest_file(&index); + else if (log_lock(&index) > 0) { + mail_index_recreate(&index); + log_unlock(&index); + } + usleep(10000); + } +} diff --git a/community-scripts/rename/rename.sh b/community-scripts/rename/rename.sh new file mode 100644 index 0000000..a73eaef --- /dev/null +++ b/community-scripts/rename/rename.sh @@ -0,0 +1,379 @@ +#!/bin/bash + +set -e +function main() +{ + mountpt="/mnt/gluster"; + + mkdir -p ${mountpt}/rename-testdir; + + cd ${mountpt}/rename-testdir; + + # TODO: get the 'ls -l' of backend also + + # case 1 + echo "============================" + echo 1 > 1; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 5; rm 5; + + echo "----------------------------" + echo 1 > 1; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5; + echo 1 > 1; + mv 1 5; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 5; rm 5; + + + # case 2 + echo "============================" + + echo 1 > 1; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 2; rm 2; + + echo "----------------------------" + + echo 1 > 1; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2; + echo 1 > 1; + mv 1 2; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 2; rm 2; + + # case 3 + echo "============================" + + echo 1 > 1; + echo 55555 > 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l + cat 5; rm 5; + + echo "----------------------------" + echo 1 > 1; + echo 55555 > 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5; + echo 1 > 1; + mv 1 5; + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l + cat 5; rm 5; + + # case 4; + echo "============================" + + echo 1 > 1; + echo 22 > 2; + mv 2 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5 + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l + cat 5; rm 5; + + echo "----------------------------" + echo 1 > 1; + echo 22 > 2; + mv 2 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 5 + echo 1 > 1; + mv 1 5 + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l + cat 5; rm 5; + + # case 5 + echo "============================" + + echo 1 > 1; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + echo hello > 1; + mv 1 2 + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 2; rm 2; + + echo "----------------------------" + echo 1 > 1; + echo 55555 > 5; + mv 5 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2 + echo 1 > 1; + mv 1 2 + ls -l /export/d*/rename-testdir + echo "on mount" + ls -l; + cat 2; rm 2; + + # case 6 + echo "============================" + + echo 1 > 1; + echo 22 > 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2; + ls -l /export/d*/rename-testdir + ls -l + cat 2; rm 2; + + + echo "----------------------------" + echo 1 > 1; + echo 22 > 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2; + echo 1 > 1; + mv 1 2; + ls -l /export/d*/rename-testdir + ls -l + cat 2; rm 2; + + # case 7 + echo "============================" + + echo 1 > 1; + echo 4444 > 4; + mv 4 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2 + ls -l /export/d*/rename-testdir + ls -l + cat 2; rm 2; + + echo "----------------------------" + echo 1 > 1; + echo 4444 > 4; + mv 4 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 1 2 + echo 1 > 1; + mv 1 2 + ls -l /export/d*/rename-testdir + ls -l + cat 2; rm 2; + + # case 8 + echo "============================" + + echo 1 > 1; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 5; + ls -l /export/d*/rename-testdir + ls -l; + cat 5; rm 5; + + echo "----------------------------" + + # case 9 + echo "============================" + + echo 1 > 1; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 3; + ls -l /export/d*/rename-testdir + ls -l; + cat 3; rm 3; + + echo "----------------------------" + + # case 10 + echo "============================" + echo 1 > 1; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 4 + ls -l /export/d*/rename-testdir + ls -l; + cat 4; rm 4; + + echo "----------------------------" + + # case 11 + echo "============================" + echo 1 > 1; + echo 55555 > 5; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 5 + ls -l /export/d*/rename-testdir + ls -l; + cat 5; rm 5; + + echo "----------------------------" + + # case 12 + echo "============================" + echo 1 > 1; + echo 333 > 3; + mv 1 2; mv 3 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 5 + ls -l /export/d*/rename-testdir + ls -l; + cat 5; rm 5; + + echo "----------------------------" + + # case 13 + echo "============================" + echo 1 > 1; + echo 4444 > 4; + mv 1 2; mv 4 5; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 5 + ls -l /export/d*/rename-testdir + ls -l; + cat 5; rm 5; + + echo "----------------------------" + + # case 14 + echo "============================" + echo 1 > 1; + echo 55555 > 5; + mv 1 2; mv 5 3; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 3 + ls -l /export/d*/rename-testdir + ls -l; + cat 3; rm 3; + + echo "----------------------------" + + # case 15 + echo "============================" + echo 1 > 1; + echo 333 > 3; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 3 + ls -l /export/d*/rename-testdir + ls -l; + cat 3; rm 3; + + echo "----------------------------" + + # case 16 + echo "============================" + echo 1 > 1; + echo 4444 > 4 + mv 1 2; mv 4 3; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 3 + ls -l /export/d*/rename-testdir + ls -l; + cat 3; rm 3; + + echo "----------------------------" + + # case 17 + echo "============================" + echo 1 > 1; + echo 55555 > 5; + mv 1 2; mv 5 4; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 4 + ls -l /export/d*/rename-testdir + ls -l; + cat 4; rm 4; + + echo "----------------------------" + + # case 18 + echo "============================" + echo 1 > 1; + echo 333 > 3; + mv 1 2; mv 3 4; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 4 + ls -l /export/d*/rename-testdir + ls -l; + cat 4; rm 4; + + echo "----------------------------" + + # case 19 + echo "============================" + echo 1 > 1; + echo 4444 > 4; + mv 1 2; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 4 + ls -l /export/d*/rename-testdir + ls -l; + cat 4; rm 4; + + echo "----------------------------" + + # case 20 + echo "============================" + echo 1 > 1; + echo 7777777 > 7; + mv 1 2; mv 7 4; + echo "before" + ls -l /export/d*/rename-testdir + mv 2 4 + ls -l /export/d*/rename-testdir + ls -l; + cat 4; rm 4; + + echo "----------------------------" + +} + +main "$@" diff --git a/community-scripts/rename/repo.py b/community-scripts/rename/repo.py new file mode 100644 index 0000000..a1a6b0d --- /dev/null +++ b/community-scripts/rename/repo.py @@ -0,0 +1,103 @@ +import os +import stat +import subprocess +import md5 +import time +import exceptions + +indexdir="/mnt/gluster/testdir/" +inputdirs=[ + "/mnt/gluster/input1/", + "/mnt/gluster/input2/", + "/mnt/gluster/input3/", + "/mnt/gluster/input4/", + "/mnt/gluster/input5/"] + +def getstat(filepath): + return os.stat(filepath) + +def getmd5sum(path): + f = open(path,'rb') + m = md5.new() + while True: + data = f.read(8096) + if(not data): + break + m.update(data) + f.close() + return m.hexdigest() + +def listdir(path): + + cmd = "ls -rt " + path + filelist = [] + process = subprocess.Popen(cmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + for line in process.stdout: + #print line + line = str(line).rstrip() + filelist.append(path + str(line)) + + output,error = process.communicate() + #print output + return filelist + + +def writeindex(indexfilepath,map): + + tempfile = indexfilepath + ".tmp" + fd = open(tempfile, 'w') + for fname in map.keys(): + lst = map[fname] + print >> fd , '%s %s' % (lst[0], lst[1]) + + fd.flush() + fd.close() + os.rename(tempfile, indexfilepath) + +def loadindex(indexfilepath): + try: + f = open(indexfilepath,'r') + lst = [] + for line in f: + lst.append(line) + f.close() + except Exception,e: + print e,indexfilepath + +while(True): + + for dir in inputdirs: + map = {} + ret = listdir(dir) + fname = dir.split("/")[-2] + idxname = fname + ".idx" + print "dir = " + str(dir) + for x in ret: + sts = getstat(x) + m5 = getmd5sum(x) + lst = [m5,sts] + map[x] = lst + if os.path.exists(indexdir + "/" + idxname): + loadindex(indexdir + "/" + idxname) + writeindex(indexdir + "/" + idxname,map) + + met = listdir(indexdir) + for z in met: + loadindex(z) + mapx={} + metname = indexdir + "/meta.idx" + for y in met: + sts = getstat(y) + m5 = getmd5sum(y) + lst = [m5,sts] + mapx[y] = lst + writeindex(metname,mapx) + + print "sleeping for 60 secs" + time.sleep(10) + +#listdir(inputdirs[0]) + + + -- cgit