From 1dbe9a05feac5032990457058f7cef686a293973 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Wed, 20 Feb 2013 14:11:36 -0500 Subject: glusterd: allow multiple instances of glusterd on one machine This is needed to support automated testing of cluster-communication features such as probing and quorum. In order to use this, you need to do the following preparatory steps. * Copy /var/lib/glusterd to another directory for each virtual host * Ensure that each virtual host has a different UUID in its glusterd.info Now you can start each copy of glusterd with the following xlator-options. * management.transport.socket.bind-address=$ip_address * management.working-directory=$unique_working_directory You can use 127.x.y.z addresses for binding without needing to assign them to interfaces explicitly. Note that you must use addresses, not names, because of some stuff in the socket code that's not worth fixing just for this usage, but after that you can use names in /etc/hosts instead. At this point you can issue CLI commands to a specific glusterd using the --remote-host option. So far probe, volume create/start/stop, mount, and basic I/O all seem to work as expected with multiple instances. Change-Id: I1beabb44cff8763d2774bc208b2ffcda27c1a550 BUG: 913555 Signed-off-by: Jeff Darcy Reviewed-on: http://review.gluster.org/4556 Tested-by: Gluster Build System Reviewed-by: Anand Avati --- tests/bugs/bug-913555.t | 66 ++++++++++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-handler.c | 26 ++++++++++- xlators/mgmt/glusterd/src/glusterd-utils.c | 48 ++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-utils.h | 3 ++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100755 tests/bugs/bug-913555.t diff --git a/tests/bugs/bug-913555.t b/tests/bugs/bug-913555.t new file mode 100755 index 00000000000..0e08bd377ae --- /dev/null +++ b/tests/bugs/bug-913555.t @@ -0,0 +1,66 @@ +#!/bin/bash + +# Test that a volume becomes unwritable when the cluster loses quorum. + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +function vglusterd { + wd=$1/wd-$2 + cp -r /var/lib/glusterd $wd + rm -rf $wd/peers/* $wd/vols/* + echo -n "UUID=$(uuidgen)\noperating-version=1\n" > $wd/glusterd.info + opt1="management.transport.socket.bind-address=127.0.0.$2" + opt2="management.working-directory=$wd" + glusterd --xlator-option $opt1 --xlator-option $opt2 +} + +function check_fs { + df $1 &> /dev/null + echo $? +} + +function check_peers { + $VCLI peer status | grep 'Peer in Cluster (Connected)' | wc -l +} + +cleanup; + +topwd=$(mktemp -d) +trap "rm -rf $topwd" EXIT + +vglusterd $topwd 100 +VCLI="$CLI --remote-host=127.0.0.100" +vglusterd $topwd 101 +TEST $VCLI peer probe 127.0.0.101 +vglusterd $topwd 102 +TEST $VCLI peer probe 127.0.0.102 + +EXPECT_WITHIN 20 2 check_peers + +create_cmd="$VCLI volume create $V0" +for i in $(seq 100 102); do + mkdir -p $B0/$V0$i + create_cmd="$create_cmd 127.0.0.$i:$B0/$V0$i" +done + +TEST $create_cmd +TEST $VCLI volume set $V0 cluster.server-quorum-type server +TEST $VCLI volume start $V0 +TEST glusterfs --volfile-server=127.0.0.100 --volfile-id=$V0 $M0 + +# Kill one pseudo-node, make sure the others survive and volume stays up. +kill -9 $(ps -ef | grep gluster | grep 127.0.0.102 | awk '{print $2}') +EXPECT_WITHIN 20 1 check_peers +fs_status=$(check_fs $M0) +nnodes=$(pidof glusterfsd | wc -w) +TEST [ "$fs_status" = 0 -a "$nnodes" = 2 ] + +# Kill another pseudo-node, make sure the last one dies and volume goes down. +kill -9 $(ps -ef | grep gluster | grep 127.0.0.101 | awk '{print $2}') +EXPECT_WITHIN 20 0 check_peers +fs_status=$(check_fs $M0) +nnodes=$(pidof glusterfsd | wc -w) +TEST [ "$fs_status" = 1 -a "$nnodes" = 0 ] + +cleanup diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 7d39c1e0785..04e5833b732 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -707,6 +707,7 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req) glusterd_peerinfo_t *peerinfo = NULL; gf_boolean_t run_fsm = _gf_true; xlator_t *this = NULL; + char *bind_name = NULL; GF_ASSERT (req); this = THIS; @@ -736,7 +737,16 @@ glusterd_handle_cli_probe (rpcsvc_request_t *req) gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d", cli_req.hostname, cli_req.port); - if (glusterd_is_local_addr(cli_req.hostname)) { + if (dict_get_str(this->options,"transport.socket.bind-address", + &bind_name) == 0) { + gf_log ("glusterd", GF_LOG_DEBUG, + "only checking probe address vs. bind address"); + ret = glusterd_is_same_address(bind_name,cli_req.hostname); + } + else { + ret = glusterd_is_local_addr(cli_req.hostname); + } + if (ret) { glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST, NULL, cli_req.hostname, cli_req.port); ret = 0; @@ -2434,6 +2444,7 @@ glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo, dict_t *options = NULL; int ret = -1; glusterd_peerctx_t *peerctx = NULL; + data_t *data = NULL; peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t); if (!peerctx) @@ -2450,6 +2461,19 @@ glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo, if (ret) goto out; + /* + * For simulated multi-node testing, we need to make sure that we + * create our RPC endpoint with the same address that the peer would + * use to reach us. + */ + if (this->options) { + data = dict_get(this->options,"transport.socket.bind-address"); + if (data) { + ret = dict_set(options, + "transport.socket.source-addr",data); + } + } + ret = glusterd_rpc_create (&peerinfo->rpc, options, glusterd_peer_rpc_notify, peerctx); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 7d64a2a0810..54c095eec48 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -7396,3 +7396,51 @@ glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key) return 0; } + +gf_boolean_t +glusterd_is_same_address (char *name1, char *name2) +{ + struct addrinfo *addr1 = NULL; + struct addrinfo *addr2 = NULL; + struct addrinfo *p = NULL; + struct addrinfo *q = NULL; + gf_boolean_t ret = _gf_false; + int gai_err = 0; + + gai_err = getaddrinfo(name1,NULL,NULL,&addr1); + if (gai_err != 0) { + gf_log (name1, GF_LOG_WARNING, + "error in getaddrinfo: %s\n", gai_strerror(gai_err)); + goto out; + } + + gai_err = getaddrinfo(name2,NULL,NULL,&addr2); + if (gai_err != 0) { + gf_log (name2, GF_LOG_WARNING, + "error in getaddrinfo: %s\n", gai_strerror(gai_err)); + goto out; + } + + for (p = addr1; p; p = p->ai_next) { + for (q = addr2; q; q = q->ai_next) { + if (p->ai_addrlen != q->ai_addrlen) { + continue; + } + if (memcmp(p->ai_addr,q->ai_addr,p->ai_addrlen)) { + continue; + } + ret = _gf_true; + goto out; + } + } + +out: + if (addr1) { + freeaddrinfo(addr1); + } + if (addr2) { + freeaddrinfo(addr2); + } + return ret; + +} diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 1f84634582b..b6a8675b223 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -509,4 +509,7 @@ glusterd_generate_and_set_task_id (dict_t *dict, char *key); int glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key); + +gf_boolean_t +glusterd_is_same_address (char *name1, char *name2); #endif -- cgit