diff options
author | Jeff Darcy <jdarcy@redhat.com> | 2014-04-22 07:30:00 -0700 |
---|---|---|
committer | Gerrit Code Review <review@dev.gluster.org> | 2014-04-22 07:30:00 -0700 |
commit | 66909ef9da737371395e63fd2557f118f6672446 (patch) | |
tree | cf3856de0695ef9b1abf371449dc8e9cf40a261c | |
parent | 9e95331b04a0dd7a1f2ddac9982faee53d9fee11 (diff) | |
parent | 927483b6fbf2a8e731f9e34dd70dc4179accc079 (diff) |
Merge "nsr: use different etcd keys for each NSR subvolume"
-rwxr-xr-x | tests/basic/four-brick.t | 85 | ||||
-rw-r--r-- | xlators/cluster/nsr-recon/src/recon_driver.c | 1 | ||||
-rw-r--r-- | xlators/cluster/nsr-server/src/leader.c | 12 | ||||
-rw-r--r-- | xlators/cluster/nsr-server/src/nsr-internal.h | 5 | ||||
-rw-r--r-- | xlators/cluster/nsr-server/src/nsr.c | 51 | ||||
-rw-r--r-- | xlators/cluster/nsr-server/src/recon_notify.c | 2 | ||||
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.c | 20 | ||||
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd.h | 3 |
8 files changed, 156 insertions, 23 deletions
diff --git a/tests/basic/four-brick.t b/tests/basic/four-brick.t new file mode 100755 index 000000000..a8d9cd400 --- /dev/null +++ b/tests/basic/four-brick.t @@ -0,0 +1,85 @@ +#!/bin/bash + +# Test *very basic* NSR functionality - startup, mount, simplest possible file +# write. + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +function get_rep_count { + v=$(getfattr --only-values -e text -n trusted.nsr.rep-count $1 2> /dev/null) + #echo $v > /dev/tty + echo $v +} + +function ping_file { + dd if=/dev/urandom of=$1 bs=4k count=100 2> /dev/null +} + +function kill_brick { + bpid=$(cat /var/lib/glusterd/vols/${V0}/run/*-${V0}${1}.pid) + rpid=$(cat /var/lib/glusterd/vols/${V0}/run/*-${V0}${1}-recon.pid) + echo "brick PID = $bpid" > /dev/tty + echo "recon PID = $rpid" > /dev/tty + kill -9 $bpid $rpid +} + +function count_matches { + n=0 + for f in $B0/$V0[1234]/$1; do + cmp $M0/$1 $f 2> /dev/null + if [ $? = 0 ]; then + n=$((n+1)) + fi + done + echo $n +} + +cleanup + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume info + +TEST mkdir -p ${V0}{1,2,3,4} +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4} + +EXPECT "$V0" volinfo_field $V0 'Volume Name' +EXPECT 'Created' volinfo_field $V0 'Status' +EXPECT '4' brick_count $V0 + +TEST $CLI volume set $V0 cluster.nsr on +TEST $CLI volume set $V0 cluster.nsr.recon on + +TEST $CLI volume start $V0 +EXPECT 'Started' volinfo_field $V0 'Status' + +## Mount FUSE with caching disabled (read-only) +TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0 + +# Give the bricks a chance to connect to each other. +EXPECT_WITHIN 10 "2" get_rep_count $M0 + +TEST ping_file $M0/probe +TEST cmp ${M0}/probe ${B0}/${V0}1/probe +TEST cmp ${M0}/probe ${B0}/${V0}2/probe + +# Kill one brick from each pair. +TEST kill_brick 1 +TEST kill_brick 3 +sleep 10 + +# Make sure only one copy makes it while degraded. +TEST ping_file $M0/probe2 +TEST [ $(count_matches probe2) = 1 ] + +# Restart the brick and give reconciliation a chance to run. +# TBD: figure out why reconciliation takes so $#@! long to run +TEST $CLI volume start $V0 force +sleep 20 + +# Make sure *both* copies are valid after reconciliation. +TEST [ $(count_matches probe2) = 2 ] + +cleanup +#killall -9 etcd diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c index b02abf2bb..e0327f81d 100644 --- a/xlators/cluster/nsr-recon/src/recon_driver.c +++ b/xlators/cluster/nsr-recon/src/recon_driver.c @@ -3066,6 +3066,7 @@ nsr_reconciliation_driver(void *arg) nsr_driver_log (this->name, GF_LOG_INFO, "finished recon work as joiner \n"); + break; default: nsr_driver_log (this->name, GF_LOG_ERROR, diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c index 944c85cdc..02a2609c8 100644 --- a/xlators/cluster/nsr-server/src/leader.c +++ b/xlators/cluster/nsr-server/src/leader.c @@ -30,7 +30,6 @@ #include "../../nsr-recon/src/recon_driver.h" #include "../../nsr-recon/src/recon_xlator.h" -#define NSR_KEY "xyzzy" #define NSR_TTL 5 static void @@ -39,20 +38,19 @@ nsr_set_leader (xlator_t *this, etcd_session etcd) long term = 0; etcd_result res; nsr_private_t *priv = this->private; - char *term_key = priv->term_uuid; char n_t[sizeof(long)+1]; char *text = NULL; gf_log (this->name, GF_LOG_INFO, "Just became leader"); - text = etcd_get(etcd, priv->term_uuid); + text = etcd_get(etcd, priv->term_key); if(text == NULL) { term = 0; } else { term = strtol(text, NULL, 10); } sprintf(n_t,"%ld",term+1); - res = etcd_set(etcd, term_key,n_t,text,0); + res = etcd_set(etcd, priv->term_key,n_t,text,0); if(res != ETCD_OK) { gf_log (this->name, GF_LOG_ERROR, "failed to set term"); return; @@ -84,7 +82,7 @@ nsr_leader_thread (void *arg) char *index_out = NULL; gf_log (this->name, GF_LOG_INFO, - "calling glfs_open_str on servers %s", priv->etcd_servers); + "calling etcd_open_str on servers %s", priv->etcd_servers); priv->etcd = etcd_open_str(priv->etcd_servers); if (!(priv->etcd)) { @@ -98,7 +96,7 @@ nsr_leader_thread (void *arg) for (;;) { /* Not leader yet. Try to become leader. */ for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (res == ETCD_OK) { break; @@ -113,7 +111,7 @@ nsr_leader_thread (void *arg) index_in = index_out; index_out = NULL; for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (index_out && (index_in != index_out)) { if (index_in) { diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h index 7825973f7..fc612c136 100644 --- a/xlators/cluster/nsr-server/src/nsr-internal.h +++ b/xlators/cluster/nsr-server/src/nsr-internal.h @@ -35,8 +35,9 @@ typedef struct _nsr_recon_notify_ev_s { typedef struct { char *etcd_servers; - char *vol_uuid; - char *term_uuid; + char *subvol_uuid; + char *leader_key; + char *term_key; char *brick_uuid; gf_boolean_t leader; uint8_t up_children; diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c index 214c54f18..eda9e555a 100644 --- a/xlators/cluster/nsr-server/src/nsr.c +++ b/xlators/cluster/nsr-server/src/nsr.c @@ -500,6 +500,25 @@ out: extern void *nsr_leader_thread (void *); +void +nsr_deallocate_priv (nsr_private_t *priv) +{ + if (!priv) { + return; + } + + if (priv->leader_key) { + GF_FREE(priv->leader_key); + } + + if (priv->term_key) { + GF_FREE(priv->term_key); + } + + GF_FREE(priv); +} + + int32_t nsr_init (xlator_t *this) { @@ -571,12 +590,22 @@ nsr_init (xlator_t *this) gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???"); goto err; } - priv->vol_uuid = "temporary"; + + GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err); + gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid); + if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate leader key"); + goto err; + } + if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate term key"); + goto err; + } uuid_generate(tmp_uuid); priv->brick_uuid = strdup(uuid_utoa(tmp_uuid)); - priv->term_uuid = "nsr-term"; - gf_log (this->name, GF_LOG_INFO, - "brick_uuid = %s\n", priv->brick_uuid); + gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid); GF_OPTION_INIT ("my-name", my_name, str, err); if (!my_name) { @@ -707,7 +736,10 @@ nsr_init (xlator_t *this) sleep(1); } - (void)pthread_create(&kid,NULL,nsr_leader_thread,this); + if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to start leader thread"); + } while (priv->leader_inited == 0) { sleep(1); } @@ -726,9 +758,7 @@ nsr_init (xlator_t *this) return 0; err: - if (priv) { - GF_FREE(priv); - } + nsr_deallocate_priv(priv); return -1; } @@ -736,6 +766,7 @@ err: void nsr_fini (xlator_t *this) { + nsr_deallocate_priv(this->private); } class_methods_t class_methods = { @@ -765,5 +796,9 @@ struct volume_options options[] = { .type = GF_OPTION_TYPE_STR, .description = "list of comma seperated etc servers" }, + { .key = {"subvol-uuid"}, + .type = GF_OPTION_TYPE_STR, + .description = "UUID for this NSR (sub)volume" + }, { .key = {NULL} }, }; diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c index 7397192ae..24f7cf2de 100644 --- a/xlators/cluster/nsr-server/src/recon_notify.c +++ b/xlators/cluster/nsr-server/src/recon_notify.c @@ -234,7 +234,7 @@ nsr_setup_recon (xlator_t *this) if (priv->nsr_recon_start == _gf_false) return 0; - ctx->fs = glfs_new(priv->vol_uuid); + ctx->fs = glfs_new(priv->subvol_uuid); if (!ctx->fs) { ret = 1; gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n"); diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index aee655733..8a94309ed 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -1431,9 +1431,14 @@ assign_groups (glusterd_volinfo_t *volinfo) glusterd_brickinfo_t *brickinfo = NULL; uint16_t group_num = 0; int in_group = 0; + uuid_t tmp_uuid; list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + if (in_group == 0) { + uuid_generate(tmp_uuid); + } brickinfo->group = group_num; + uuid_copy(brickinfo->nsr_uuid,tmp_uuid); if (++in_group >= volinfo->replica_count) { in_group = 0; ++group_num; @@ -1469,7 +1474,6 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, gf_boolean_t enable_recon = _gf_false; static uint32_t nsr_port = 27000; - if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr.recon") > 0) { enable_recon = _gf_true; } @@ -1520,6 +1524,10 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, return -1; if (xlator_set_option(me,"leader",leader_opt)) return -1; + if (xlator_set_option(me,"subvol-uuid", + uuid_utoa(brickinfo->nsr_uuid))) { + return -1; + } #define FILL_REMOTE_NAMES { \ strcat(remote_names, \ @@ -1586,7 +1594,7 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, if (enable_recon == _gf_false) return 0; - /* Now fill in the various files required for reeconciliation */ + /* Now fill in the various files required for reconciliation */ snprintf (filepath, PATH_MAX, "%s-nsr-recon.vol", dst); @@ -1898,8 +1906,8 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, /* TBD: conditionalize on NSR being enabled */ if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { - assign_groups(volinfo); - ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, changelog_basepath); + ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, + changelog_basepath); if (ret) { return -1; } @@ -3774,6 +3782,10 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo) } } + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + assign_groups(volinfo); + } + list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { gf_log ("", GF_LOG_DEBUG, "Found a brick - %s:%s", brickinfo->hostname, diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 2b9e0787d..3d810ed97 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -198,7 +198,8 @@ struct glusterd_brickinfo { * more efficient too, though it would require some further adaptation * to support more than one layer of hierarchy. */ - uint16_t group; + uint16_t group; + uuid_t nsr_uuid; }; typedef struct glusterd_brickinfo glusterd_brickinfo_t; |