diff options
author | Lakshmipathi <lakshmipathi.g@gmail.com> | 2011-07-08 11:51:53 +0530 |
---|---|---|
committer | Lakshmipathi <lakshmipathi.g@gmail.com> | 2011-07-08 11:51:53 +0530 |
commit | b47528cc4bb7a308c74d4dae7c2adcf9f7392f4e (patch) | |
tree | 35742effa1325ea42a99492167d16e75feabd373 /community-scripts | |
parent | bb2ca55002b8faac2b4687a97b611724081e601b (diff) |
Added community provided scripts for lock and rename
Diffstat (limited to 'community-scripts')
-rw-r--r-- | community-scripts/locks/lock.php | 22 | ||||
-rw-r--r-- | community-scripts/locks/locky.sh | 10 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034.tar | bin | 0 -> 20480 bytes | |||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/README | 88 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/gt.h | 12 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/gt_init.c | 114 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/gt_read.c | 131 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/gt_rename.c | 86 | ||||
-rw-r--r-- | community-scripts/rename/atomic/bug1034/volume.info | 10 | ||||
-rw-r--r-- | community-scripts/rename/atomic/reader.c | 23 | ||||
-rw-r--r-- | community-scripts/rename/atomic/writer.c | 48 | ||||
-rw-r--r-- | community-scripts/rename/index.c | 333 | ||||
-rw-r--r-- | community-scripts/rename/rename.sh | 379 | ||||
-rw-r--r-- | community-scripts/rename/repo.py | 103 |
14 files changed, 1359 insertions, 0 deletions
diff --git a/community-scripts/locks/lock.php b/community-scripts/locks/lock.php new file mode 100644 index 0000000..dc4d039 --- /dev/null +++ b/community-scripts/locks/lock.php @@ -0,0 +1,22 @@ +#/usr/bin/php +<? + +$fh = fopen('gluster.test', 'ab+'); +echo('Opened.'."\n"); +sleep(2); +echo('Shared lock attempt.'."\n"); +flock($fh, LOCK_SH); +echo('Locked as shared.'."\n"); +sleep(10); +echo('Exclusive lock attempt.'."\n"); +flock($fh, LOCK_EX); +echo('Locked exclusively.'."\n"); +sleep(10); +flock($fh, LOCK_UN); +echo('Unlocked.'."\n"); +sleep(2); +fclose($fh); +echo('Closed.'."\n"); +sleep(1); + +?> diff --git a/community-scripts/locks/locky.sh b/community-scripts/locks/locky.sh new file mode 100644 index 0000000..8d9518b --- /dev/null +++ b/community-scripts/locks/locky.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +for((i = 0; i < 200; ++i)); do + flock counter -c ' + read value < counter + echo $value + value=$(( value + 1 )) + echo ${value} > counter + ' + done diff --git a/community-scripts/rename/atomic/bug1034.tar b/community-scripts/rename/atomic/bug1034.tar Binary files differnew file mode 100644 index 0000000..88b5eb5 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034.tar diff --git a/community-scripts/rename/atomic/bug1034/README b/community-scripts/rename/atomic/bug1034/README new file mode 100644 index 0000000..fd0acd7 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/README @@ -0,0 +1,88 @@ +*** Cluster and Gluster Configuration + +The cluster has MAX nodes. Each node has a gluster brick. +The bricks are assembled into a single gluster volume, using +pure replication. The gluster volume is mounted at the same +BASE_PATH on each node. + +Edit BASE_PATH in gt.h to the appropriate path before +compiling. + +Compile the test programs + cc gt_init.c -o gt_init + cc gt_rename.c -o gt_rename + cc gt_read.c -o gt_read +And distribute them to all nodes of the cluster. + +*** Test Configuration + +There are three test programs. + +One instance of gt_init is run once, on any node in +the cluster, before the other test processes are started. +This cleans up and initializes the contents of BASE_PATH, +then terminates. + +An instance of gt_rename runs on every node. Each instance is +started with a numeral from 1 to MAX as a command line argument. +These continuously replace a set of files by renaming a temp +file on top of existing files. + +An instance of gt_read runs on every node. Each instance is +started with a numeral from 1 to MAX as a command line argument. +These continuously read all the files touched by the set of +gt_rename processes. + +I.e., with cluster nodes numbered 1 to MAX, on node n start: + gt_rename n & + gt_read MAX & + +*** File Structure + +BASE_PATH/1/WA_RC_0 + /WA_RC_1 + /WA_RC_2 + /... + /MAX/WA_RC_0 + /WA_RC_1 + /WA_RC_2 + +There is a directory under BASE_PATH for each node, +numbered 1 through MAX. + +gt_rename processes manipulate files only within +their node's directory. + +gt_read test process reads all files and writes no +file. + +Every file is written by only one test process. +The contents of every directory (not directory tree) +is written by only one test process. + +Every file is read by MAX test processes other than +the single writing test process. + +*** Test Operation + +The gt_init process destroys any existing file structure +under BASE_PATH, then creates and initializes the contents +of the file structure. + +The gt_rename processes always write a fixed distinctive +value into each file they write. They open a temp file, write a +fixed value, and rename the file onto the standard file. +The fixed value is different for every standard file. + +The gt_rename and gt_read processes always test each file for +existence, then check the contents for the expected distinctive +value, for each file they access. Apparently missing files or +unexpected contents are logged. + +The gt_rename and gt_read processes iterate every CYCLE_TIME ++[0..CYCLE_JITTER) milliseconds until terminated. + +By observing the output of the gt_read processes, the +characteristics of bug1034 can be seen. On node n, gt_read +gets ENOENT for open and for read-after-successful-open on +BASE_PATH/n/*. diff --git a/community-scripts/rename/atomic/bug1034/gt.h b/community-scripts/rename/atomic/bug1034/gt.h new file mode 100644 index 0000000..da6a57c --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt.h @@ -0,0 +1,12 @@ +#ifndef _BUG1034_TEST_ +#define _BUG1034_TEST_ + +// #define BASE_PATH "/common_mount_point/test_directory" +#define BASE_PATH "/sme_cluster/global/bug1034_test" + +#define CYCLE_TIME 5 +#define CYCLE_JITTER 5 + +#define SUBDIR_FILES 3 + +#endif /* ! _BUG1034_TEST_ */ diff --git a/community-scripts/rename/atomic/bug1034/gt_init.c b/community-scripts/rename/atomic/bug1034/gt_init.c new file mode 100644 index 0000000..f3dadf0 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_init.c @@ -0,0 +1,114 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node_count\n", progname); + exit(1); +} + +// rm -rf +static void +delete_tree(char *rootpath) +{ + int retval; + struct stat statbuf; + + retval = lstat(rootpath, &statbuf); + if (retval < 0) { + return; + } + + if (S_ISDIR(statbuf.st_mode)) { + DIR *dir; + struct dirent *dent; + char pathbuf[PATH_MAX]; + + dir = opendir(rootpath); + while ((dent = readdir(dir)) != NULL) { + if (0 == strcmp(".", dent->d_name)) { + continue; + } + if (0 == strcmp("..", dent->d_name)) { + continue; + } + snprintf(pathbuf, sizeof(pathbuf), + "%s/%s", rootpath, dent->d_name); + delete_tree(pathbuf); + } + closedir(dir); + + (void)rmdir(rootpath); + } else { + (void)unlink(rootpath); + } +} + +static void +create_subdir(char *dirpath) +{ + int i; + + mkdir(dirpath, 0775); + + for (i = 0; i < SUBDIR_FILES; i++) { + char pathbuf[PATH_MAX]; + int fd; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/WA_RC_%d", dirpath, i); + fd = open(pathbuf, O_RDWR|O_CREAT, 0664); + write(fd, pathbuf, strlen(pathbuf)); + close(fd); + } +} + +static void +create_tree(char *rootpath, int node_count) +{ + int n; + + mkdir(rootpath, 0775); + + for (n = 1; n <= node_count; n++) { + char pathbuf[PATH_MAX]; + int fd; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d", rootpath, n); + mkdir(pathbuf, 0775); + create_subdir(pathbuf); + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node_count; + + if (argc != 2) { + usage(argv[0]); + } + + node_count = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + delete_tree(BASE_PATH); + + create_tree(BASE_PATH, node_count); + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/gt_read.c b/community-scripts/rename/atomic/bug1034/gt_read.c new file mode 100644 index 0000000..7d07d59 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_read.c @@ -0,0 +1,131 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <ctype.h> + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node_count\n", progname); + exit(1); +} + +static void +dump_data(char *buf, size_t buflen) +{ + int i; + + fprintf(stderr, "raw data: "); + for (i = 0; i < buflen; i++) { + int c = buf[i]; + + if (isgraph(c)) { + fprintf(stderr, "%c", c); + } else if (isspace(c)) { + switch (c) { + case ' ': + fprintf(stderr, " "); + break; + case '\f': + fprintf(stderr, "\f"); + break; + case '\n': + fprintf(stderr, "\n"); + break; + case '\r': + fprintf(stderr, "\r"); + break; + case '\t': + fprintf(stderr, "\t"); + break; + case '\v': + fprintf(stderr, "\v"); + break; + default: + fprintf(stderr, "\%03o", c); + break; + } + } else { + fprintf(stderr, "\%03o", c); + } + } + fprintf(stderr, "\n"); +} + +static void +touch_tree(char *rootpath, int node_count) +{ + char pathbuf[PATH_MAX]; + char databuf[PATH_MAX]; + int fd; + int node; + int i; + int retval; + + for (node = 1; node <= node_count; node++) { + for (i = 0; i < SUBDIR_FILES; i++) { + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d/WA_RC_%d", rootpath, node, i); + fd = open(pathbuf, O_RDONLY); + if (0 > fd) { + fprintf(stderr, "gt_read: %s: open %s\n", + strerror(errno), pathbuf); + continue; + } + retval = read(fd, databuf, sizeof(databuf)); + close(fd); + if (0 > retval) { + fprintf(stderr, "gt_read: %s: read %s\n", + strerror(errno), pathbuf); + } else if (retval != strlen(pathbuf)) { + fprintf(stderr, + "gt_read: incomplete read: read %s\n", + pathbuf); + } else { + databuf[retval] = '\0'; + if (0 != strcmp(pathbuf, databuf)) { + fprintf(stderr, + "gt_read: bad data: read %s\n", + pathbuf); + dump_data(databuf, strlen(pathbuf)); + } + } + } + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node_count; + + if (argc != 2) { + usage(argv[0]); + } + + node_count = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + while (1) { + int snooze; + + touch_tree(BASE_PATH, node_count); + + snooze = CYCLE_TIME + random()%CYCLE_JITTER; + usleep(1000*snooze); + } + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/gt_rename.c b/community-scripts/rename/atomic/bug1034/gt_rename.c new file mode 100644 index 0000000..96ecce0 --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/gt_rename.c @@ -0,0 +1,86 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +#include "gt.h" + +static void +usage(char *progname) +{ + fprintf(stderr, "usage: %s node\n", progname); + exit(1); +} + +static void +touch_tree(char *rootpath, int node) +{ + char pathbuf[PATH_MAX]; + char databuf[PATH_MAX]; + int fd; + int i; + int retval; + + for (i = 0; i < SUBDIR_FILES; i++) { + char tempbuf[PATH_MAX]; + + snprintf(pathbuf, sizeof(pathbuf), + "%s/%d/WA_RC_%d", rootpath, node, i); + snprintf(tempbuf, sizeof(tempbuf), + "%s/%d/WA_RC_%d.temp", rootpath, node, i); + fd = open(tempbuf, O_RDWR|O_CREAT, 0644); + if (0 > fd) { + fprintf(stderr, "gt_rename: %s: open %s\n", + strerror(errno), tempbuf); + continue; + } + retval = write(fd, pathbuf, strlen(pathbuf)); + if (0 > retval) { + fprintf(stderr, "gt_rename: %s: write %s\n", + strerror(errno), pathbuf); + } else if (retval != strlen(pathbuf)) { + fprintf(stderr, "gt_rename: incomplete write: write %s\n", + pathbuf); + } + close(fd); + + retval = rename(tempbuf, pathbuf); + if (0 > retval) { + fprintf(stderr, "gt_rename: %s rename %s\n", + strerror(errno), pathbuf); + } + } +} + +int +main(int argc, char **argv) +{ + char *arg_ptr; + long node; + + if (argc != 2) { + usage(argv[0]); + } + + node = strtol(argv[1], &arg_ptr, 10); + if (*arg_ptr != '\0') { + usage(argv[0]); + } + + while (1) { + int snooze; + + touch_tree(BASE_PATH, node); + + snooze = CYCLE_TIME + random()%CYCLE_JITTER; + usleep(1000*snooze); + } + + exit(0); +} diff --git a/community-scripts/rename/atomic/bug1034/volume.info b/community-scripts/rename/atomic/bug1034/volume.info new file mode 100644 index 0000000..68d654e --- /dev/null +++ b/community-scripts/rename/atomic/bug1034/volume.info @@ -0,0 +1,10 @@ +type=2 +count=2 +status=1 +sub_count=2 +version=1 +transport-type=0 +volume-id=01234567-89ab-cdef-0123-456789abcdef +brick-0=host0:-our-brick-location +brick-1=host1:-our-brick-location + diff --git a/community-scripts/rename/atomic/reader.c b/community-scripts/rename/atomic/reader.c new file mode 100644 index 0000000..d25d9d8 --- /dev/null +++ b/community-scripts/rename/atomic/reader.c @@ -0,0 +1,23 @@ +/*
+ gcc reader.c -o reader -Wall
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+int main(void)
+{
+ int fd;
+
+ for (;;) {
+ usleep(rand() % 1000);
+ fd = open("dovecot.index", O_RDONLY);
+ if (fd == -1) {
+ perror("open(dovecot.index)");
+ break;
+ }
+ close(fd);
+ }
+ return 0;
+}
diff --git a/community-scripts/rename/atomic/writer.c b/community-scripts/rename/atomic/writer.c new file mode 100644 index 0000000..16d07e1 --- /dev/null +++ b/community-scripts/rename/atomic/writer.c @@ -0,0 +1,48 @@ +/*
+ gcc writer.c -o writer -Wall
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+int main(void)
+{
+ int fd_old, fd_new, ret;
+
+ for (;;) {
+ fd_old = open("dovecot.index", O_RDWR);
+ if (fd_old == -1) {
+ perror("open(dovecot.index");
+ break;
+ }
+
+ usleep(rand() % 1000);
+ fd_new = creat("dovecot.index.tmp", 0600);
+ if (fd_new == -1) {
+ perror("creat(dovecot.index.tmp)");
+ break;
+ }
+ write(fd_new, "foo", 3);
+ close(fd_new);
+
+ ret = link("dovecot.index", "dovecot.index.backup.tmp");
+ if (ret < 0) {
+ perror("link(dovecot.index, dovecot.index.backup.tmp)");
+ break;
+ }
+ if (rename("dovecot.index.backup.tmp", "dovecot.index.backup") < 0) {
+ perror("rename(dovecot.index.backup.tmp, dovecot.index.backup)");
+ break;
+ }
+
+ usleep(rand() % 1000);
+ if (rename("dovecot.index.tmp", "dovecot.index") < 0) {
+ perror("rename(dovecot.index.tmp, dovecot.index)");
+ break;
+ }
+ usleep(rand() % 1000);
+ close(fd_old);
+ }
+ return 0;
+}
diff --git a/community-scripts/rename/index.c b/community-scripts/rename/index.c new file mode 100644 index 0000000..4382095 --- /dev/null +++ b/community-scripts/rename/index.c @@ -0,0 +1,333 @@ +/*
+ gcc index.c -o index -Wall -g
+ ./index &
+ ./index &
+ ./index &
+
+ abort()s on failure, runs forever on non-failure
+*/
+#define _XOPEN_SOURCE 500 /* for pread() */
+#define _BSD_SOURCE /* for major(), minor() */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+struct mail_index {
+ const char *filepath;
+ int fd;
+ int log_fd;
+};
+
+struct mail_index_header {
+ /* major version is increased only when you can't have backwards
+ compatibility. minor version is increased when header size is
+ increased to contain new non-critical fields. */
+ uint8_t major_version;
+ uint8_t minor_version;
+
+ uint16_t base_header_size;
+ uint32_t header_size; /* base + extended header size */
+ uint32_t record_size;
+
+ uint8_t compat_flags; /* enum mail_index_header_compat_flags */
+ uint8_t unused[3];
+
+ uint32_t indexid;
+ uint32_t flags;
+
+ uint32_t uid_validity;
+ uint32_t next_uid;
+
+ uint32_t messages_count;
+ uint32_t unused_old_recent_messages_count;
+ uint32_t seen_messages_count;
+ uint32_t deleted_messages_count;
+
+ uint32_t first_recent_uid;
+ /* these UIDs may not exist and may not even be unseen/deleted */
+ uint32_t first_unseen_uid_lowwater;
+ uint32_t first_deleted_uid_lowwater;
+
+ uint32_t log_file_seq;
+ /* non-external records between tail..head haven't been committed to
+ mailbox yet. */
+ uint32_t log_file_tail_offset;
+ uint32_t log_file_head_offset;
+
+ uint64_t sync_size;
+ uint32_t sync_stamp;
+
+ /* daily first UIDs that have been added to index. */
+ uint32_t day_stamp;
+ uint32_t day_first_uid[8];
+};
+
+struct mail_index_record {
+ uint32_t uid;
+ uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */
+};
+int mail_index_try_open_only(struct mail_index *index)
+{
+ index->fd = open(index->filepath, O_RDWR);
+ if (index->fd == -1) {
+ if (errno != ENOENT)
+ abort();
+
+ /* have to create it */
+ return 0;
+ }
+ return 1;
+}
+
+void mail_index_close_file(struct mail_index *index)
+{
+ if (index->fd != -1) {
+ if (close(index->fd) < 0)
+ abort();
+ index->fd = -1;
+ }
+}
+
+int mail_index_reopen_if_changed(struct mail_index *index)
+{
+ struct stat st1, st2;
+
+ if (index->fd == -1)
+ return mail_index_try_open_only(index);
+
+ //nfs_flush_file_handle_cache(index->filepath);
+ if (stat(index->filepath, &st2) < 0)
+ abort();
+
+#define CMP_DEV_T(a, b) (major(a) == major(b) && minor(a) == minor(b))
+ if (fstat(index->fd, &st1) < 0) {
+ abort();
+ } else if (st1.st_ino == st2.st_ino &&
+ CMP_DEV_T(st1.st_dev, st2.st_dev)) {
+ /* the same file */
+ return 1;
+ }
+
+ /* new file */
+ mail_index_close_file(index);
+ return mail_index_try_open_only(index);
+}
+
+static int mail_index_read_header(struct mail_index *index,
+ void *buf, size_t buf_size, size_t *pos_r)
+{
+ size_t pos;
+ int ret;
+
+ memset(buf, 0, sizeof(struct mail_index_header));
+
+ pos = 0;
+ do {
+ ret = pread(index->fd, (char*)buf + pos,
+ buf_size - pos, pos);
+ if (ret <= 0)
+ abort();
+ if (ret > 0)
+ pos += ret;
+ } while (ret > 0 && pos < sizeof(struct mail_index_header));
+
+ *pos_r = pos;
+ return ret;
+}
+
+static int
+mail_index_read_map(struct mail_index *index, off_t file_size)
+{
+ const struct mail_index_header *hdr;
+ unsigned char read_buf[8192];
+ const void *buf;
+ ssize_t ret;
+ size_t pos, records_size;
+ unsigned int records_count = 0, extra;
+
+ ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
+ buf = read_buf; hdr = buf;
+
+ if (ret > 0) {
+ /* header read, read the records now. */
+ records_size = (size_t)hdr->messages_count * hdr->record_size;
+ records_count = hdr->messages_count;
+
+ if (file_size - hdr->header_size < records_size ||
+ (hdr->record_size != 0 &&
+ records_size / hdr->record_size != hdr->messages_count))
+ abort();
+
+ if (pos <= hdr->header_size)
+ extra = 0;
+ else
+ extra = pos - hdr->header_size;
+ if (records_size > extra) {
+ void *data = malloc(records_size - extra);
+ ret = pread(index->fd, data, records_size - extra,
+ hdr->header_size + extra);
+ if (ret != records_size - extra) abort();
+ free(data);
+ }
+ }
+ return 1;
+}
+
+static int file_lock_do(int fd, int lock_type, int timeout_secs)
+{
+ struct flock fl;
+ int ret;
+
+ fl.l_type = lock_type;
+ fl.l_whence = SEEK_SET;
+ fl.l_start = 0;
+ fl.l_len = 0;
+
+ if (timeout_secs != 0)
+ alarm(timeout_secs);
+ ret = fcntl(fd, timeout_secs != 0 ? F_SETLKW : F_SETLK, &fl);
+ if (timeout_secs != 0)
+ alarm(0);
+
+ if (ret == 0)
+ return 1;
+
+ if (timeout_secs == 0 &&
+ (errno == EACCES || errno == EAGAIN)) {
+ /* locked by another process */
+ return 0;
+ }
+ abort();
+}
+
+static int mail_index_map_latest_file(struct mail_index *index)
+{
+ struct stat st;
+ off_t file_size;
+ int ret;
+
+ ret = mail_index_reopen_if_changed(index);
+ if (ret <= 0) {
+ if (ret < 0)
+ return -1;
+
+ /* the index file is lost/broken. */
+ return 1;
+ }
+
+ if (file_lock_do(index->fd, F_RDLCK, 120) == 0) abort();
+
+ //nfs_flush_attr_cache_fd_locked(index->filepath, index->fd);
+ if (fstat(index->fd, &st) < 0)
+ abort();
+ file_size = st.st_size;
+
+ ret = mail_index_read_map(index, file_size);
+ if (file_lock_do(index->fd, F_UNLCK, 0) == 0) abort();
+ return 1;
+}
+
+static int mail_index_create_backup(struct mail_index *index)
+{
+ char backup_path[1024], tmp_backup_path[1024];
+ int ret;
+
+ snprintf(backup_path, sizeof(backup_path), "%s.backup", index->filepath);
+ snprintf(tmp_backup_path, sizeof(tmp_backup_path), "%s.tmp", backup_path);
+ ret = link(index->filepath, tmp_backup_path);
+ if (ret < 0 && errno == EEXIST) {
+ if (unlink(tmp_backup_path) < 0 && errno != ENOENT)
+ abort();
+ ret = link(index->filepath, tmp_backup_path);
+ }
+ if (ret < 0) {
+ if (errno == ENOENT) {
+ /* no dovecot.index file, ignore */
+ return 0;
+ }
+ abort();
+ }
+
+ if (rename(tmp_backup_path, backup_path) < 0)
+ abort();
+ return 0;
+}
+
+static void mail_index_recreate(struct mail_index *index)
+{
+ struct mail_index_header hdr;
+ struct mail_index_record *recs;
+ char path[1024];
+ int fd;
+ unsigned int i, size;
+
+ snprintf(path, sizeof(path), "%s.tmp", index->filepath);
+ fd = open(path, O_RDWR|O_CREAT|O_TRUNC, 0600);
+ if (fd == -1)
+ abort();
+
+ memset(&hdr, 0, sizeof(hdr));
+ hdr.base_header_size = hdr.header_size = sizeof(hdr);
+ hdr.record_size = sizeof(*recs);
+ hdr.messages_count = (rand() % 10000) * 100;
+
+ if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) abort();
+ size = sizeof(*recs) * hdr.messages_count;
+ recs = calloc(size, 1);
+ for (i = 0; i < hdr.messages_count; i++)
+ recs[i].uid = i + 1;
+ if (write(fd, recs, size) != size) abort();
+ free(recs);
+ if (fdatasync(fd) < 0)
+ abort();
+ if (close(fd) < 0)
+ abort();
+ mail_index_create_backup(index);
+
+ if (rename(path, index->filepath) < 0) {
+ perror("rename()");
+ abort();
+ }
+}
+
+static int log_lock(struct mail_index *index)
+{
+ if (index->log_fd == -1) {
+ index->log_fd = open("/mnt/gluster/dovecot.index.log", O_CREAT | O_RDWR, 0600);
+ if (index->log_fd == -1)
+ abort();
+ }
+ return file_lock_do(index->log_fd, F_WRLCK, 0);
+}
+
+static void log_unlock(struct mail_index *index)
+{
+ if (file_lock_do(index->log_fd, F_UNLCK, 0) == 0) abort();
+}
+
+int main(void)
+{
+ struct mail_index index;
+
+ memset(&index, 0, sizeof(index));
+ index.fd = -1;
+ index.log_fd = -1;
+ index.filepath = "/mnt/gluster/dovecot.index";
+
+ for (;;) {
+ if (rand() % 100 < 70)
+ mail_index_map_latest_file(&index);
+ else if (log_lock(&index) > 0) {
+ mail_index_recreate(&index);
+ log_unlock(&index);
+ }
+ usleep(10000);
+ }
+}
diff --git a/community-scripts/rename/rename.sh b/community-scripts/rename/rename.sh new file mode 100644 index 0000000..a73eaef --- /dev/null +++ b/community-scripts/rename/rename.sh @@ -0,0 +1,379 @@ +#!/bin/bash
+
+set -e
+function main()
+{
+ mountpt="/mnt/gluster";
+
+ mkdir -p ${mountpt}/rename-testdir;
+
+ cd ${mountpt}/rename-testdir;
+
+ # TODO: get the 'ls -l' of backend also
+
+ # case 1
+ echo "============================"
+ echo 1 > 1;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 5; rm 5;
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5;
+ echo 1 > 1;
+ mv 1 5;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 5; rm 5;
+
+
+ # case 2
+ echo "============================"
+
+ echo 1 > 1;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 2; rm 2;
+
+ echo "----------------------------"
+
+ echo 1 > 1;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2;
+ echo 1 > 1;
+ mv 1 2;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 2; rm 2;
+
+ # case 3
+ echo "============================"
+
+ echo 1 > 1;
+ echo 55555 > 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l
+ cat 5; rm 5;
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo 55555 > 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5;
+ echo 1 > 1;
+ mv 1 5;
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l
+ cat 5; rm 5;
+
+ # case 4;
+ echo "============================"
+
+ echo 1 > 1;
+ echo 22 > 2;
+ mv 2 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l
+ cat 5; rm 5;
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo 22 > 2;
+ mv 2 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 5
+ echo 1 > 1;
+ mv 1 5
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l
+ cat 5; rm 5;
+
+ # case 5
+ echo "============================"
+
+ echo 1 > 1;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ echo hello > 1;
+ mv 1 2
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 2; rm 2;
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo 55555 > 5;
+ mv 5 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2
+ echo 1 > 1;
+ mv 1 2
+ ls -l /export/d*/rename-testdir
+ echo "on mount"
+ ls -l;
+ cat 2; rm 2;
+
+ # case 6
+ echo "============================"
+
+ echo 1 > 1;
+ echo 22 > 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2;
+ ls -l /export/d*/rename-testdir
+ ls -l
+ cat 2; rm 2;
+
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo 22 > 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2;
+ echo 1 > 1;
+ mv 1 2;
+ ls -l /export/d*/rename-testdir
+ ls -l
+ cat 2; rm 2;
+
+ # case 7
+ echo "============================"
+
+ echo 1 > 1;
+ echo 4444 > 4;
+ mv 4 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2
+ ls -l /export/d*/rename-testdir
+ ls -l
+ cat 2; rm 2;
+
+ echo "----------------------------"
+ echo 1 > 1;
+ echo 4444 > 4;
+ mv 4 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 1 2
+ echo 1 > 1;
+ mv 1 2
+ ls -l /export/d*/rename-testdir
+ ls -l
+ cat 2; rm 2;
+
+ # case 8
+ echo "============================"
+
+ echo 1 > 1;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 5;
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 5; rm 5;
+
+ echo "----------------------------"
+
+ # case 9
+ echo "============================"
+
+ echo 1 > 1;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 3;
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 3; rm 3;
+
+ echo "----------------------------"
+
+ # case 10
+ echo "============================"
+ echo 1 > 1;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 4
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 4; rm 4;
+
+ echo "----------------------------"
+
+ # case 11
+ echo "============================"
+ echo 1 > 1;
+ echo 55555 > 5;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 5
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 5; rm 5;
+
+ echo "----------------------------"
+
+ # case 12
+ echo "============================"
+ echo 1 > 1;
+ echo 333 > 3;
+ mv 1 2; mv 3 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 5
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 5; rm 5;
+
+ echo "----------------------------"
+
+ # case 13
+ echo "============================"
+ echo 1 > 1;
+ echo 4444 > 4;
+ mv 1 2; mv 4 5;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 5
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 5; rm 5;
+
+ echo "----------------------------"
+
+ # case 14
+ echo "============================"
+ echo 1 > 1;
+ echo 55555 > 5;
+ mv 1 2; mv 5 3;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 3
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 3; rm 3;
+
+ echo "----------------------------"
+
+ # case 15
+ echo "============================"
+ echo 1 > 1;
+ echo 333 > 3;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 3
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 3; rm 3;
+
+ echo "----------------------------"
+
+ # case 16
+ echo "============================"
+ echo 1 > 1;
+ echo 4444 > 4
+ mv 1 2; mv 4 3;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 3
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 3; rm 3;
+
+ echo "----------------------------"
+
+ # case 17
+ echo "============================"
+ echo 1 > 1;
+ echo 55555 > 5;
+ mv 1 2; mv 5 4;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 4
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 4; rm 4;
+
+ echo "----------------------------"
+
+ # case 18
+ echo "============================"
+ echo 1 > 1;
+ echo 333 > 3;
+ mv 1 2; mv 3 4;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 4
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 4; rm 4;
+
+ echo "----------------------------"
+
+ # case 19
+ echo "============================"
+ echo 1 > 1;
+ echo 4444 > 4;
+ mv 1 2;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 4
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 4; rm 4;
+
+ echo "----------------------------"
+
+ # case 20
+ echo "============================"
+ echo 1 > 1;
+ echo 7777777 > 7;
+ mv 1 2; mv 7 4;
+ echo "before"
+ ls -l /export/d*/rename-testdir
+ mv 2 4
+ ls -l /export/d*/rename-testdir
+ ls -l;
+ cat 4; rm 4;
+
+ echo "----------------------------"
+
+}
+
+main "$@"
diff --git a/community-scripts/rename/repo.py b/community-scripts/rename/repo.py new file mode 100644 index 0000000..a1a6b0d --- /dev/null +++ b/community-scripts/rename/repo.py @@ -0,0 +1,103 @@ +import os +import stat +import subprocess +import md5 +import time +import exceptions + +indexdir="/mnt/gluster/testdir/" +inputdirs=[ + "/mnt/gluster/input1/", + "/mnt/gluster/input2/", + "/mnt/gluster/input3/", + "/mnt/gluster/input4/", + "/mnt/gluster/input5/"] + +def getstat(filepath): + return os.stat(filepath) + +def getmd5sum(path): + f = open(path,'rb') + m = md5.new() + while True: + data = f.read(8096) + if(not data): + break + m.update(data) + f.close() + return m.hexdigest() + +def listdir(path): + + cmd = "ls -rt " + path + filelist = [] + process = subprocess.Popen(cmd, shell=True,stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + for line in process.stdout: + #print line + line = str(line).rstrip() + filelist.append(path + str(line)) + + output,error = process.communicate() + #print output + return filelist + + +def writeindex(indexfilepath,map): + + tempfile = indexfilepath + ".tmp" + fd = open(tempfile, 'w') + for fname in map.keys(): + lst = map[fname] + print >> fd , '%s %s' % (lst[0], lst[1]) + + fd.flush() + fd.close() + os.rename(tempfile, indexfilepath) + +def loadindex(indexfilepath): + try: + f = open(indexfilepath,'r') + lst = [] + for line in f: + lst.append(line) + f.close() + except Exception,e: + print e,indexfilepath + +while(True): + + for dir in inputdirs: + map = {} + ret = listdir(dir) + fname = dir.split("/")[-2] + idxname = fname + ".idx" + print "dir = " + str(dir) + for x in ret: + sts = getstat(x) + m5 = getmd5sum(x) + lst = [m5,sts] + map[x] = lst + if os.path.exists(indexdir + "/" + idxname): + loadindex(indexdir + "/" + idxname) + writeindex(indexdir + "/" + idxname,map) + + met = listdir(indexdir) + for z in met: + loadindex(z) + mapx={} + metname = indexdir + "/meta.idx" + for y in met: + sts = getstat(y) + m5 = getmd5sum(y) + lst = [m5,sts] + mapx[y] = lst + writeindex(metname,mapx) + + print "sleeping for 60 secs" + time.sleep(10) + +#listdir(inputdirs[0]) + + + |