From 927483b6fbf2a8e731f9e34dd70dc4179accc079 Mon Sep 17 00:00:00 2001 From: Jeff Darcy Date: Tue, 25 Mar 2014 16:48:37 +0000 Subject: nsr: use different etcd keys for each NSR subvolume This is necessary both for separate volumes using NSR and for DHT volumes composed of multiple NSR subvolumes. Change-Id: Ia269d70b535cc26900f8b6e7f22706087746fbe7 Signed-off-by: Jeff Darcy --- xlators/cluster/nsr-recon/src/recon_driver.c | 1 + xlators/cluster/nsr-server/src/leader.c | 12 +++---- xlators/cluster/nsr-server/src/nsr-internal.h | 5 +-- xlators/cluster/nsr-server/src/nsr.c | 51 ++++++++++++++++++++++----- xlators/cluster/nsr-server/src/recon_notify.c | 2 +- xlators/mgmt/glusterd/src/glusterd-volgen.c | 20 ++++++++--- xlators/mgmt/glusterd/src/glusterd.h | 3 +- 7 files changed, 71 insertions(+), 23 deletions(-) (limited to 'xlators') diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c index b02abf2bb..e0327f81d 100644 --- a/xlators/cluster/nsr-recon/src/recon_driver.c +++ b/xlators/cluster/nsr-recon/src/recon_driver.c @@ -3066,6 +3066,7 @@ nsr_reconciliation_driver(void *arg) nsr_driver_log (this->name, GF_LOG_INFO, "finished recon work as joiner \n"); + break; default: nsr_driver_log (this->name, GF_LOG_ERROR, diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c index 944c85cdc..02a2609c8 100644 --- a/xlators/cluster/nsr-server/src/leader.c +++ b/xlators/cluster/nsr-server/src/leader.c @@ -30,7 +30,6 @@ #include "../../nsr-recon/src/recon_driver.h" #include "../../nsr-recon/src/recon_xlator.h" -#define NSR_KEY "xyzzy" #define NSR_TTL 5 static void @@ -39,20 +38,19 @@ nsr_set_leader (xlator_t *this, etcd_session etcd) long term = 0; etcd_result res; nsr_private_t *priv = this->private; - char *term_key = priv->term_uuid; char n_t[sizeof(long)+1]; char *text = NULL; gf_log (this->name, GF_LOG_INFO, "Just became leader"); - text = etcd_get(etcd, priv->term_uuid); + text = etcd_get(etcd, priv->term_key); if(text == NULL) { term = 0; } else { term = strtol(text, NULL, 10); } sprintf(n_t,"%ld",term+1); - res = etcd_set(etcd, term_key,n_t,text,0); + res = etcd_set(etcd, priv->term_key,n_t,text,0); if(res != ETCD_OK) { gf_log (this->name, GF_LOG_ERROR, "failed to set term"); return; @@ -84,7 +82,7 @@ nsr_leader_thread (void *arg) char *index_out = NULL; gf_log (this->name, GF_LOG_INFO, - "calling glfs_open_str on servers %s", priv->etcd_servers); + "calling etcd_open_str on servers %s", priv->etcd_servers); priv->etcd = etcd_open_str(priv->etcd_servers); if (!(priv->etcd)) { @@ -98,7 +96,7 @@ nsr_leader_thread (void *arg) for (;;) { /* Not leader yet. Try to become leader. */ for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (res == ETCD_OK) { break; @@ -113,7 +111,7 @@ nsr_leader_thread (void *arg) index_in = index_out; index_out = NULL; for (;;) { - res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL, + res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL, index_in, &index_out); if (index_out && (index_in != index_out)) { if (index_in) { diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h index 7825973f7..fc612c136 100644 --- a/xlators/cluster/nsr-server/src/nsr-internal.h +++ b/xlators/cluster/nsr-server/src/nsr-internal.h @@ -35,8 +35,9 @@ typedef struct _nsr_recon_notify_ev_s { typedef struct { char *etcd_servers; - char *vol_uuid; - char *term_uuid; + char *subvol_uuid; + char *leader_key; + char *term_key; char *brick_uuid; gf_boolean_t leader; uint8_t up_children; diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c index 214c54f18..eda9e555a 100644 --- a/xlators/cluster/nsr-server/src/nsr.c +++ b/xlators/cluster/nsr-server/src/nsr.c @@ -500,6 +500,25 @@ out: extern void *nsr_leader_thread (void *); +void +nsr_deallocate_priv (nsr_private_t *priv) +{ + if (!priv) { + return; + } + + if (priv->leader_key) { + GF_FREE(priv->leader_key); + } + + if (priv->term_key) { + GF_FREE(priv->term_key); + } + + GF_FREE(priv); +} + + int32_t nsr_init (xlator_t *this) { @@ -571,12 +590,22 @@ nsr_init (xlator_t *this) gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???"); goto err; } - priv->vol_uuid = "temporary"; + + GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err); + gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid); + if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate leader key"); + goto err; + } + if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) { + gf_log (this->name, GF_LOG_ERROR, + "could not generate term key"); + goto err; + } uuid_generate(tmp_uuid); priv->brick_uuid = strdup(uuid_utoa(tmp_uuid)); - priv->term_uuid = "nsr-term"; - gf_log (this->name, GF_LOG_INFO, - "brick_uuid = %s\n", priv->brick_uuid); + gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid); GF_OPTION_INIT ("my-name", my_name, str, err); if (!my_name) { @@ -707,7 +736,10 @@ nsr_init (xlator_t *this) sleep(1); } - (void)pthread_create(&kid,NULL,nsr_leader_thread,this); + if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to start leader thread"); + } while (priv->leader_inited == 0) { sleep(1); } @@ -726,9 +758,7 @@ nsr_init (xlator_t *this) return 0; err: - if (priv) { - GF_FREE(priv); - } + nsr_deallocate_priv(priv); return -1; } @@ -736,6 +766,7 @@ err: void nsr_fini (xlator_t *this) { + nsr_deallocate_priv(this->private); } class_methods_t class_methods = { @@ -765,5 +796,9 @@ struct volume_options options[] = { .type = GF_OPTION_TYPE_STR, .description = "list of comma seperated etc servers" }, + { .key = {"subvol-uuid"}, + .type = GF_OPTION_TYPE_STR, + .description = "UUID for this NSR (sub)volume" + }, { .key = {NULL} }, }; diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c index 7397192ae..24f7cf2de 100644 --- a/xlators/cluster/nsr-server/src/recon_notify.c +++ b/xlators/cluster/nsr-server/src/recon_notify.c @@ -234,7 +234,7 @@ nsr_setup_recon (xlator_t *this) if (priv->nsr_recon_start == _gf_false) return 0; - ctx->fs = glfs_new(priv->vol_uuid); + ctx->fs = glfs_new(priv->subvol_uuid); if (!ctx->fs) { ret = 1; gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n"); diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index aee655733..8a94309ed 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -1431,9 +1431,14 @@ assign_groups (glusterd_volinfo_t *volinfo) glusterd_brickinfo_t *brickinfo = NULL; uint16_t group_num = 0; int in_group = 0; + uuid_t tmp_uuid; list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { + if (in_group == 0) { + uuid_generate(tmp_uuid); + } brickinfo->group = group_num; + uuid_copy(brickinfo->nsr_uuid,tmp_uuid); if (++in_group >= volinfo->replica_count) { in_group = 0; ++group_num; @@ -1469,7 +1474,6 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, gf_boolean_t enable_recon = _gf_false; static uint32_t nsr_port = 27000; - if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr.recon") > 0) { enable_recon = _gf_true; } @@ -1520,6 +1524,10 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, return -1; if (xlator_set_option(me,"leader",leader_opt)) return -1; + if (xlator_set_option(me,"subvol-uuid", + uuid_utoa(brickinfo->nsr_uuid))) { + return -1; + } #define FILL_REMOTE_NAMES { \ strcat(remote_names, \ @@ -1586,7 +1594,7 @@ add_nsr_stuff (volgen_graph_t *graph, char *volname, if (enable_recon == _gf_false) return 0; - /* Now fill in the various files required for reeconciliation */ + /* Now fill in the various files required for reconciliation */ snprintf (filepath, PATH_MAX, "%s-nsr-recon.vol", dst); @@ -1898,8 +1906,8 @@ server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo, /* TBD: conditionalize on NSR being enabled */ if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { - assign_groups(volinfo); - ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, changelog_basepath); + ret = add_nsr_stuff (graph, volname, brickinfo, volinfo, + changelog_basepath); if (ret) { return -1; } @@ -3774,6 +3782,10 @@ generate_brick_volfiles (glusterd_volinfo_t *volinfo) } } + if (glusterd_volinfo_get_boolean(volinfo,"cluster.nsr") > 0) { + assign_groups(volinfo); + } + list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) { gf_log ("", GF_LOG_DEBUG, "Found a brick - %s:%s", brickinfo->hostname, diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 2b9e0787d..3d810ed97 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -198,7 +198,8 @@ struct glusterd_brickinfo { * more efficient too, though it would require some further adaptation * to support more than one layer of hierarchy. */ - uint16_t group; + uint16_t group; + uuid_t nsr_uuid; }; typedef struct glusterd_brickinfo glusterd_brickinfo_t; -- cgit