diff options
Diffstat (limited to 'tests/basic/fencing')
| -rw-r--r-- | tests/basic/fencing/afr-lock-heal-advanced.c | 227 | ||||
| -rw-r--r-- | tests/basic/fencing/afr-lock-heal-advanced.t | 115 | ||||
| -rw-r--r-- | tests/basic/fencing/afr-lock-heal-basic.c | 182 | ||||
| -rw-r--r-- | tests/basic/fencing/afr-lock-heal-basic.t | 102 | ||||
| -rw-r--r-- | tests/basic/fencing/fence-basic.c | 229 | ||||
| -rwxr-xr-x | tests/basic/fencing/fence-basic.t | 31 | ||||
| -rw-r--r-- | tests/basic/fencing/fencing-crash-conistency.t | 62 | ||||
| -rw-r--r-- | tests/basic/fencing/test-fence-option.t | 37 |
8 files changed, 985 insertions, 0 deletions
diff --git a/tests/basic/fencing/afr-lock-heal-advanced.c b/tests/basic/fencing/afr-lock-heal-advanced.c new file mode 100644 index 00000000000..e202ccd5b29 --- /dev/null +++ b/tests/basic/fencing/afr-lock-heal-advanced.c @@ -0,0 +1,227 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <signal.h> +#include <unistd.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +#define GF_ENFORCE_MANDATORY_LOCK "trusted.glusterfs.enforce-mandatory-lock" + +FILE *logfile_fp; + +#define LOG_ERR(func, err) \ + do { \ + if (!logfile_fp) { \ + fprintf(stderr, "%\n%d %s : returned error (%s)\n", __LINE__, \ + func, strerror(err)); \ + fflush(stderr); \ + } else { \ + fprintf(logfile_fp, "\n%d %s : returned error (%s)\n", __LINE__, \ + func, strerror(err)); \ + fflush(logfile_fp); \ + } \ + } while (0) + +glfs_t * +setup_client(char *hostname, char *volname, char *log_file) +{ + int ret = 0; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + fprintf(logfile_fp, "\nglfs_new: returned NULL (%s)\n", + strerror(errno)); + goto error; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_set_volfile_server failed ret:%d (%s)\n", + ret, strerror(errno)); + goto error; + } + + ret = glfs_set_logging(fs, log_file, 7); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_set_logging failed with ret: %d (%s)\n", + ret, strerror(errno)); + goto error; + } + + ret = glfs_init(fs); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_init failed with ret: %d (%s)\n", ret, + strerror(errno)); + goto error; + } + +out: + return fs; +error: + return NULL; +} + +glfs_fd_t * +open_file(glfs_t *fs, char *fname) +{ + glfs_fd_t *fd = NULL; + + fd = glfs_creat(fs, fname, O_CREAT, 0644); + if (!fd) { + LOG_ERR("glfs_creat", errno); + goto out; + } +out: + return fd; +} + +int +acquire_mandatory_lock(glfs_t *fs, glfs_fd_t *fd) +{ + struct flock lock; + int ret = 0; + + /* initialize lock */ + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 100; + + ret = glfs_fsetxattr(fd, GF_ENFORCE_MANDATORY_LOCK, "set", 8, 0); + if (ret < 0) { + LOG_ERR("glfs_fsetxattr", errno); + ret = -1; + goto out; + } + + /* take a write mandatory lock */ + ret = glfs_file_lock(fd, F_SETLKW, &lock, GLFS_LK_MANDATORY); + if (ret) { + LOG_ERR("glfs_file_lock", errno); + ret = -1; + goto out; + } + +out: + return ret; +} + +int +perform_test(glfs_t *fs, char *file1, char *file2) +{ + int ret = 0; + glfs_fd_t *fd1 = NULL; + glfs_fd_t *fd2 = NULL; + char *buf = "0123456789"; + + fd1 = open_file(fs, file1); + if (!fd1) { + ret = -1; + goto out; + } + fd2 = open_file(fs, file2); + if (!fd2) { + ret = -1; + goto out; + } + + /* Kill one brick from the .t.*/ + pause(); + + ret = acquire_mandatory_lock(fs, fd1); + if (ret) { + goto out; + } + ret = acquire_mandatory_lock(fs, fd2); + if (ret) { + goto out; + } + + /* Bring the brick up and let the locks heal. */ + pause(); + /*At this point, the .t would have killed and brought back 2 bricks, marking + * the fd bad.*/ + + ret = glfs_write(fd1, buf, 10, 0); + if (ret > 0) { + /* Write is supposed to fail with EBADFD*/ + LOG_ERR("glfs_write", ret); + goto out; + } + + ret = 0; +out: + if (fd1) + glfs_close(fd1); + if (fd2) + glfs_close(fd2); + return ret; +} + +static void +sigusr1_handler(int signo) +{ + /*Signal caught. Just continue with the execution.*/ +} + +int +main(int argc, char *argv[]) +{ + int ret = 0; + glfs_t *fs = NULL; + char *volname = NULL; + char log_file[100]; + char *hostname = NULL; + char *fname1 = NULL; + char *fname2 = NULL; + + if (argc != 7) { + fprintf(stderr, + "Expect following args %s <host> <volname> <file1> <file2> " + "<log file " + "location> <log_file_suffix>\n", + argv[0]); + return -1; + } + + hostname = argv[1]; + volname = argv[2]; + fname1 = argv[3]; + fname2 = argv[4]; + + /*Use SIGUSR1 and pause()as a means of hitting break-points this program + *when signalled from the .t test case.*/ + if (signal(SIGUSR1, sigusr1_handler) == SIG_ERR) { + LOG_ERR("SIGUSR1 handler error", errno); + exit(EXIT_FAILURE); + } + + sprintf(log_file, "%s/%s.%s.%s", argv[5], "lock-heal.c", argv[6], "log"); + logfile_fp = fopen(log_file, "w"); + if (!logfile_fp) { + fprintf(stderr, "\nfailed to open %s\n", log_file); + fflush(stderr); + return -1; + } + + sprintf(log_file, "%s/%s.%s.%s", argv[5], "glfs-client", argv[6], "log"); + fs = setup_client(hostname, volname, log_file); + if (!fs) { + LOG_ERR("setup_client", errno); + return -1; + } + + ret = perform_test(fs, fname1, fname2); + +error: + if (fs) { + /*glfs_fini(fs)*/; // glfs fini path is racy and crashes the program + } + + fclose(logfile_fp); + + return ret; +} diff --git a/tests/basic/fencing/afr-lock-heal-advanced.t b/tests/basic/fencing/afr-lock-heal-advanced.t new file mode 100644 index 00000000000..8a5b5989b5e --- /dev/null +++ b/tests/basic/fencing/afr-lock-heal-advanced.t @@ -0,0 +1,115 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; +PROCESS_UP_TIMEOUT=90 + +function is_gfapi_program_alive() +{ + pid=$1 + ps -p $pid + if [ $? -eq 0 ] + then + echo "Y" + else + echo "N" + fi +} + +function get_active_lock_count { + brick=$1 + i1=$2 + i2=$3 + pattern="ACTIVE.*client-${brick: -1}" + + sdump=$(generate_brick_statedump $V0 $H0 $brick) + lock_count1="$(egrep "$i1" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + lock_count2="$(egrep "$i2" $sdump -A3| egrep "$pattern"|uniq|wc -l)" + echo "$((lock_count1+lock_count2))" +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} +EXPECT 'Created' volinfo_field $V0 'Status'; +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume set $V0 locks.mandatory-locking forced +TEST $CLI volume set $V0 enforce-mandatory-lock on +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +logdir=`gluster --print-logdir` +TEST build_tester $(dirname $0)/afr-lock-heal-advanced.c -lgfapi -ggdb + +#------------------------------------------------------------------------------ +# Use more than 1 fd from same client so that list_for_each_* loops are executed more than once. +$(dirname $0)/afr-lock-heal-advanced $H0 $V0 "/FILE1" "/FILE2" $logdir C1& +client_pid=$! +TEST [ $client_pid ] + +TEST sleep 5 # By now, the client would have opened an fd on FILE1 and FILE2 and waiting for a SIGUSR1. +EXPECT "Y" is_gfapi_program_alive $client_pid + +gfid_str1=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE1)) +inode1="FILE1|gfid:$gfid_str1" +gfid_str2=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE2)) +inode2="FILE2|gfid:$gfid_str2" + +# Kill brick-3 and let client-1 take lock on both files. +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill -SIGUSR1 $client_pid +# If program is still alive, glfs_file_lock() was a success. +EXPECT "Y" is_gfapi_program_alive $client_pid + +# Check lock is present on brick-1 and brick-2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}1 $inode1 $inode2 + +# Restart brick-3 and check that the lock has healed on it. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 +TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. + +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}2 $inode1 $inode2 + +#------------------------------------------------------------------------------ +# Kill same brick before heal completes the first time and check it completes the second time. +TEST $CLI volume set $V0 delay-gen locks +TEST $CLI volume set $V0 delay-gen.delay-duration 5000000 +TEST $CLI volume set $V0 delay-gen.delay-percentage 100 +TEST $CLI volume set $V0 delay-gen.enable finodelk + +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST $CLI volume reset $V0 delay-gen +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "2" get_active_lock_count $B0/${V0}0 $inode1 $inode2 + +#------------------------------------------------------------------------------ +# Kill 2 bricks and bring it back. The fds must be marked bad. +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill_brick $V0 $H0 $B0/${V0}1 +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}1 + +# TODO: `gluster v statedump $V0 client localhost:$client_pid` is not working, +# so sleep for 20 seconds for the client to connect to connect to the bricks. +TEST sleep $CHILD_UP_TIMEOUT + +# Try to write to FILE1 from the .c; it must fail. +TEST kill -SIGUSR1 $client_pid +wait $client_pid +ret=$? +TEST [ $ret == 0 ] + +cleanup_tester $(dirname $0)/afr-lock-heal-advanced +cleanup; diff --git a/tests/basic/fencing/afr-lock-heal-basic.c b/tests/basic/fencing/afr-lock-heal-basic.c new file mode 100644 index 00000000000..768c9e57181 --- /dev/null +++ b/tests/basic/fencing/afr-lock-heal-basic.c @@ -0,0 +1,182 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <signal.h> +#include <unistd.h> +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> + +#define GF_ENFORCE_MANDATORY_LOCK "trusted.glusterfs.enforce-mandatory-lock" + +FILE *logfile_fp; + +#define LOG_ERR(func, err) \ + do { \ + if (!logfile_fp) { \ + fprintf(stderr, "%\n%d %s : returned error (%s)\n", __LINE__, \ + func, strerror(err)); \ + fflush(stderr); \ + } else { \ + fprintf(logfile_fp, "\n%d %s : returned error (%s)\n", __LINE__, \ + func, strerror(err)); \ + fflush(logfile_fp); \ + } \ + } while (0) + +glfs_t * +setup_client(char *hostname, char *volname, char *log_file) +{ + int ret = 0; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + fprintf(logfile_fp, "\nglfs_new: returned NULL (%s)\n", + strerror(errno)); + goto error; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_set_volfile_server failed ret:%d (%s)\n", + ret, strerror(errno)); + goto error; + } + + ret = glfs_set_logging(fs, log_file, 7); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_set_logging failed with ret: %d (%s)\n", + ret, strerror(errno)); + goto error; + } + + ret = glfs_init(fs); + if (ret < 0) { + fprintf(logfile_fp, "\nglfs_init failed with ret: %d (%s)\n", ret, + strerror(errno)); + goto error; + } + +out: + return fs; +error: + return NULL; +} + +int +acquire_mandatory_lock(glfs_t *fs, char *fname) +{ + struct flock lock; + int ret = 0; + glfs_fd_t *fd = NULL; + + fd = glfs_creat(fs, fname, O_CREAT, 0644); + if (!fd) { + if (errno != EEXIST) { + LOG_ERR("glfs_creat", errno); + ret = -1; + goto out; + } + fd = glfs_open(fs, fname, O_RDWR | O_NONBLOCK); + if (!fd) { + LOG_ERR("glfs_open", errno); + ret = -1; + goto out; + } + } + + /* initialize lock */ + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 100; + + ret = glfs_fsetxattr(fd, GF_ENFORCE_MANDATORY_LOCK, "set", 8, 0); + if (ret < 0) { + LOG_ERR("glfs_fsetxattr", errno); + ret = -1; + goto out; + } + + pause(); + + /* take a write mandatory lock */ + ret = glfs_file_lock(fd, F_SETLKW, &lock, GLFS_LK_MANDATORY); + if (ret) { + LOG_ERR("glfs_file_lock", errno); + goto out; + } + + pause(); + +out: + if (fd) { + glfs_close(fd); + } + + return ret; +} + +static void +sigusr1_handler(int signo) +{ + /*Signal caught. Just continue with the execution.*/ +} + +int +main(int argc, char *argv[]) +{ + int ret = 0; + glfs_t *fs = NULL; + char *volname = NULL; + char log_file[100]; + char *hostname = NULL; + char *fname = NULL; + + if (argc != 6) { + fprintf(stderr, + "Expect following args %s <host> <volname> <file> <log file " + "location> <log_file_suffix>\n", + argv[0]); + return -1; + } + + hostname = argv[1]; + volname = argv[2]; + fname = argv[3]; + + /*Use SIGUSR1 and pause()as a means of hitting break-points this program + *when signalled from the .t test case.*/ + if (signal(SIGUSR1, sigusr1_handler) == SIG_ERR) { + LOG_ERR("SIGUSR1 handler error", errno); + exit(EXIT_FAILURE); + } + + sprintf(log_file, "%s/%s.%s.%s", argv[4], "lock-heal-basic.c", argv[5], + "log"); + logfile_fp = fopen(log_file, "w"); + if (!logfile_fp) { + fprintf(stderr, "\nfailed to open %s\n", log_file); + fflush(stderr); + return -1; + } + + sprintf(log_file, "%s/%s.%s.%s", argv[4], "glfs-client", argv[5], "log"); + fs = setup_client(hostname, volname, log_file); + if (!fs) { + LOG_ERR("setup_client", errno); + return -1; + } + + ret = acquire_mandatory_lock(fs, fname); + +error: + if (fs) { + /*glfs_fini(fs)*/; // glfs fini path is racy and crashes the program + } + + fclose(logfile_fp); + + return ret; +} diff --git a/tests/basic/fencing/afr-lock-heal-basic.t b/tests/basic/fencing/afr-lock-heal-basic.t new file mode 100644 index 00000000000..69131af085d --- /dev/null +++ b/tests/basic/fencing/afr-lock-heal-basic.t @@ -0,0 +1,102 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +function is_gfapi_program_alive() +{ + pid=$1 + ps -p $pid + if [ $? -eq 0 ] + then + echo "Y" + else + echo "N" + fi +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2} +EXPECT 'Created' volinfo_field $V0 'Status'; +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 performance.open-behind off +TEST $CLI volume set $V0 locks.mandatory-locking forced +TEST $CLI volume set $V0 enforce-mandatory-lock on +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +logdir=`gluster --print-logdir` +TEST build_tester $(dirname $0)/afr-lock-heal-basic.c -lgfapi -ggdb + +$(dirname $0)/afr-lock-heal-basic $H0 $V0 "/FILE" $logdir C1& +client1_pid=$! +TEST [ $client1_pid ] + +$(dirname $0)/afr-lock-heal-basic $H0 $V0 "/FILE" $logdir C2& +client2_pid=$! +TEST [ $client2_pid ] + +TEST sleep 5 # By now, the 2 clients would have opened an fd on FILE and waiting for a SIGUSR1. +EXPECT "Y" is_gfapi_program_alive $client1_pid +EXPECT "Y" is_gfapi_program_alive $client2_pid + +gfid_str=$(gf_gfid_xattr_to_str $(gf_get_gfid_xattr $B0/${V0}0/FILE)) +inode="FILE|gfid:$gfid_str" + +# Kill brick-3 and let client-1 take lock on the file. +TEST kill_brick $V0 $H0 $B0/${V0}2 +TEST kill -SIGUSR1 $client1_pid +# If program is still alive, glfs_file_lock() was a success. +EXPECT "Y" is_gfapi_program_alive $client1_pid + +# Check lock is present on brick-1 and brick-2 +b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) +c1_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) +c1_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b2" ] + +# Restart brick-3 and check that the lock has healed on it. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}2 +TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. Also wait for lock heal. + +b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) +c1_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +TEST [ "$c1_lock_on_b1" == "$c1_lock_on_b3" ] + +# Kill brick-1 and let client-2 preempt the lock on bricks 2 and 3. +TEST kill_brick $V0 $H0 $B0/${V0}0 +TEST kill -SIGUSR1 $client2_pid +# If program is still alive, glfs_file_lock() was a success. +EXPECT "Y" is_gfapi_program_alive $client2_pid + +# Restart brick-1 and let lock healing complete. +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" brick_up_status $V0 $H0 $B0/${V0}0 +TEST sleep 10 #Needed for client to re-open fd? Otherwise client_pre_lk_v2() fails with EBADFD for remote-fd. Also wait for lock heal. + +# Check that all bricks now have locks from client 2 only. +b1_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}0) +c2_lock_on_b1="$(egrep "$inode" $b1_sdump -A3| egrep 'ACTIVE.*client-0'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +b2_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}1) +c2_lock_on_b2="$(egrep "$inode" $b2_sdump -A3| egrep 'ACTIVE.*client-1'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +b3_sdump=$(generate_brick_statedump $V0 $H0 $B0/${V0}2) +c2_lock_on_b3="$(egrep "$inode" $b3_sdump -A3| egrep 'ACTIVE.*client-2'| uniq| awk '{print $1,$2,$3,S4,$5,$6,$7,$8}'|tr -d '(,), ,')" +TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b2" ] +TEST [ "$c2_lock_on_b1" == "$c2_lock_on_b3" ] +TEST [ "$c2_lock_on_b1" != "$c1_lock_on_b1" ] + +#Let the client programs run and exit. +TEST kill -SIGUSR1 $client1_pid +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "N" is_gfapi_program_alive $client1_pid +TEST kill -SIGUSR1 $client2_pid +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "N" is_gfapi_program_alive $client2_pid + +cleanup_tester $(dirname $0)/afr-lock-heal-basic +cleanup; diff --git a/tests/basic/fencing/fence-basic.c b/tests/basic/fencing/fence-basic.c new file mode 100644 index 00000000000..4aa452e19b0 --- /dev/null +++ b/tests/basic/fencing/fence-basic.c @@ -0,0 +1,229 @@ +#include <glusterfs/api/glfs.h> +#include <glusterfs/api/glfs-handles.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#define NO_INIT 1 +#define GF_ENFORCE_MANDATORY_LOCK "trusted.glusterfs.enforce-mandatory-lock" + +FILE *fp; +char *buf = "0123456789"; + +#define LOG_ERR(func, err) \ + do { \ + if (!fp) { \ + fprintf(stderr, "%\n%d %s : returned error (%s)\n", __LINE__, \ + func, strerror(err)); \ + fflush(stderr); \ + } else { \ + fprintf(fp, "\n%d %s : returned error (%s)\n", __LINE__, func, \ + strerror(err)); \ + fflush(fp); \ + } \ + } while (0) + +glfs_t * +setup_new_client(char *hostname, char *volname, char *log_file, int flag) +{ + int ret = 0; + glfs_t *fs = NULL; + + fs = glfs_new(volname); + if (!fs) { + fprintf(fp, "\nglfs_new: returned NULL (%s)\n", strerror(errno)); + goto error; + } + + ret = glfs_set_volfile_server(fs, "tcp", hostname, 24007); + if (ret < 0) { + fprintf(fp, "\nglfs_set_volfile_server failed ret:%d (%s)\n", ret, + strerror(errno)); + goto error; + } + + ret = glfs_set_logging(fs, log_file, 7); + if (ret < 0) { + fprintf(fp, "\nglfs_set_logging failed with ret: %d (%s)\n", ret, + strerror(errno)); + goto error; + } + + if (flag == NO_INIT) + goto out; + + ret = glfs_init(fs); + if (ret < 0) { + fprintf(fp, "\nglfs_init failed with ret: %d (%s)\n", ret, + strerror(errno)); + goto error; + } + +out: + return fs; +error: + return NULL; +} + +/* test plan + * + * - take mandatory lock from client 1 + * - preempt mandatory lock from client 2 + * - write from client 1 which should fail + */ + +int +test(glfs_t *fs1, glfs_t *fs2, char *fname) +{ + struct flock lock; + int ret = 0; + glfs_fd_t *fd1, *fd2 = NULL; + + fd1 = glfs_creat(fs1, fname, O_RDWR, 0777); + if (ret) { + LOG_ERR("glfs_creat", errno); + ret = -1; + goto out; + } + + fd2 = glfs_open(fs2, fname, O_RDWR | O_NONBLOCK); + if (ret) { + LOG_ERR("glfs_open", errno); + ret = -1; + goto out; + } + + /* initialize lock */ + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 100; + + ret = glfs_fsetxattr(fd1, GF_ENFORCE_MANDATORY_LOCK, "set", 8, 0); + if (ret < 0) { + LOG_ERR("glfs_fsetxattr", errno); + ret = -1; + goto out; + } + + /* take a write mandatory lock */ + ret = glfs_file_lock(fd1, F_SETLKW, &lock, GLFS_LK_MANDATORY); + if (ret) { + LOG_ERR("glfs_file_lock", errno); + goto out; + } + + ret = glfs_write(fd1, buf, 10, 0); + if (ret != 10) { + LOG_ERR("glfs_write", errno); + ret = -1; + goto out; + } + + /* write should fail */ + ret = glfs_write(fd2, buf, 10, 0); + if (ret != -1) { + LOG_ERR("glfs_write", errno); + ret = -1; + goto out; + } + + /* preempt mandatory lock from client 1*/ + ret = glfs_file_lock(fd2, F_SETLKW, &lock, GLFS_LK_MANDATORY); + if (ret) { + LOG_ERR("glfs_file_lock", errno); + goto out; + } + + /* write should succeed from client 2 */ + ret = glfs_write(fd2, buf, 10, 0); + if (ret == -1) { + LOG_ERR("glfs_write", errno); + goto out; + } + + /* write should fail from client 1 */ + ret = glfs_write(fd1, buf, 10, 0); + if (ret == 10) { + LOG_ERR("glfs_write", errno); + ret = -1; + goto out; + } + + ret = 0; + +out: + if (fd1) { + glfs_close(fd1); + } + + if (fd2) { + glfs_close(fd2); + } + + return ret; +} + +int +main(int argc, char *argv[]) +{ + int ret = 0; + glfs_t *fs1 = NULL; + glfs_t *fs2 = NULL; + char *volname = NULL; + char log_file[100]; + char *hostname = NULL; + char *fname = "/file"; + glfs_fd_t *fd1 = NULL; + glfs_fd_t *fd2 = NULL; + + if (argc != 4) { + fprintf( + stderr, + "Expect following args %s <hostname> <Vol> <log file location>\n", + argv[0]); + return -1; + } + + hostname = argv[1]; + volname = argv[2]; + + sprintf(log_file, "%s/%s", argv[3], "fence-basic.log"); + fp = fopen(log_file, "w"); + if (!fp) { + fprintf(stderr, "\nfailed to open %s\n", log_file); + fflush(stderr); + return -1; + } + + sprintf(log_file, "%s/%s", argv[3], "glfs-client-1.log"); + fs1 = setup_new_client(hostname, volname, log_file, 0); + if (!fs1) { + LOG_ERR("setup_new_client", errno); + return -1; + } + + sprintf(log_file, "%s/%s", argv[3], "glfs-client-2.log"); + fs2 = setup_new_client(hostname, volname, log_file, 0); + if (!fs2) { + LOG_ERR("setup_new_client", errno); + ret = -1; + goto error; + } + + ret = test(fs1, fs2, fname); + +error: + if (fs1) { + glfs_fini(fs1); + } + + if (fs2) { + glfs_fini(fs2); + } + + fclose(fp); + + return ret; +} diff --git a/tests/basic/fencing/fence-basic.t b/tests/basic/fencing/fence-basic.t new file mode 100755 index 00000000000..30f379e7b20 --- /dev/null +++ b/tests/basic/fencing/fence-basic.t @@ -0,0 +1,31 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 $H0:$B0/brick1 +EXPECT 'Created' volinfo_field $V0 'Status'; + +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; + +TEST $CLI volume set $V0 diagnostics.client-log-flush-timeout 30 +TEST $CLI volume set $V0 performance.write-behind off +TEST $CLI volume set $V0 locks.mandatory-locking forced +TEST $CLI volume set $V0 enforce-mandatory-lock on + + +logdir=`gluster --print-logdir` + +TEST build_tester $(dirname $0)/fence-basic.c -lgfapi -ggdb +TEST $(dirname $0)/fence-basic $H0 $V0 $logdir + +cleanup_tester $(dirname $0)/fence-basic + +cleanup;
\ No newline at end of file diff --git a/tests/basic/fencing/fencing-crash-conistency.t b/tests/basic/fencing/fencing-crash-conistency.t new file mode 100644 index 00000000000..0c69411e90c --- /dev/null +++ b/tests/basic/fencing/fencing-crash-conistency.t @@ -0,0 +1,62 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +# with lock enforcement flag write should fail with out lock + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 $H0:$B0/${V0}1 +EXPECT 'Created' volinfo_field $V0 'Status'; +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST $CLI volume set $V0 performance.write-behind off +TEST glusterfs -s $H0 --volfile-id $V0 $M0 + +TEST touch $M0/file + +#write should pass +TEST "echo "test" > $M0/file" +TEST "truncate -s 0 $M0/file" + +#enable mandatory locking +TEST $CLI volume set $V0 locks.mandatory-locking forced +TEST $CLI volume set $V0 enforce-mandatory-lock on + +#write should pass +TEST "echo "test" >> $M0/file" +TEST "truncate -s 0 $M0/file" + +#enforce lock on the file +TEST setfattr -n trusted.glusterfs.enforce-mandatory-lock -v 1 $M0/file + +#write should fail +TEST ! "echo "test" >> $M0/file" +TEST ! "truncate -s 0 $M0/file" + +#remove lock enforcement flag +TEST setfattr -x trusted.glusterfs.enforce-mandatory-lock $M0/file + +#write should pass +TEST "echo "test" >> $M0/file" +TEST "truncate -s 0 $M0/file" + +#enforce lock on the file +TEST setfattr -n trusted.glusterfs.enforce-mandatory-lock -v 1 $M0/file +#kill brick +TEST kill_brick $V0 $H0 $B0/${V0}1 + +TEST $CLI volume start $V0 force + +# wait one second for the brick to come online +sleep 2 +#write should fail (lock xlator gets lock enforcement info from disk) +TEST ! "echo "test" >> $M0/file" +TEST ! "truncate -s 0 $M0/file" + +cleanup;
\ No newline at end of file diff --git a/tests/basic/fencing/test-fence-option.t b/tests/basic/fencing/test-fence-option.t new file mode 100644 index 00000000000..115cbe7dbdf --- /dev/null +++ b/tests/basic/fencing/test-fence-option.t @@ -0,0 +1,37 @@ +#!/bin/bash + +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc + +cleanup; + +# with lock enforcement flag write should fail with out lock + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info; + +TEST $CLI volume create $V0 $H0:$B0/${V0}1 +EXPECT 'Created' volinfo_field $V0 'Status'; +TEST $CLI volume start $V0; +EXPECT 'Started' volinfo_field $V0 'Status'; +TEST glusterfs -s $H0 --volfile-id $V0 $M0 + +TEST touch $M0/file + +#setfattr for mandatory-enforcement will fail +TEST ! setfattr -n trusted.glusterfs.enforce-mandatory-lock -v 1 $M0/file + +#enable mandatory locking +TEST $CLI volume set $V0 locks.mandatory-locking forced + +#setfattr will fail +TEST ! setfattr -n trusted.glusterfs.enforce-mandatory-lock -v 1 $M0/file + +#set lock-enforcement option +TEST $CLI volume set $V0 enforce-mandatory-lock on + +#setfattr should succeed +TEST setfattr -n trusted.glusterfs.enforce-mandatory-lock -v 1 $M0/file + +cleanup;
\ No newline at end of file |