From e0cce4cf7c22d5cd8ab6c2aff4ecf28c18c6a469 Mon Sep 17 00:00:00 2001 From: Raghavan P Date: Fri, 3 Jan 2014 16:09:04 +0530 Subject: Changes to NSR reconciliation code. Following is list of changes: 1) Simulation of etcd using a file as a registry protected using locks. 2) Implement notifications for child up and child down. 3) Join a new brick into quorum. 4) add support for proper fencing and draining of IO required for reconciliaiton 5) misc changes and addressed review comments. Change-Id: Iddd1137ad6205252ed03301888bb1e83fa2221e0 Signed-off-by: Raghavan P --- xlators/cluster/nsr-server/src/nsr.c | 45 ++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) (limited to 'xlators/cluster/nsr-server/src/nsr.c') diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c index 3707b3003..f85368456 100644 --- a/xlators/cluster/nsr-server/src/nsr.c +++ b/xlators/cluster/nsr-server/src/nsr.c @@ -258,7 +258,6 @@ nsr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name, dict_t *xdata) { dict_t *result; - uint8_t up; nsr_private_t *priv = this->private; if (!priv->leader) { @@ -279,8 +278,8 @@ nsr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc, goto dn_failed; } - up = nsr_count_up_kids(this->private); - if (dict_set_uint32(result,NSR_REP_COUNT_XATTR,up) != 0) { + priv->up_children = nsr_count_up_kids(this->private); + if (dict_set_uint32(result,NSR_REP_COUNT_XATTR,priv->up_children) != 0) { goto dsu_failed; } @@ -399,6 +398,8 @@ nsr_reconfigure (xlator_t *this, dict_t *options) nsr_private_t *priv = this->private; GF_OPTION_RECONF ("leader", priv->leader, options, bool, err); + gf_log (this->name, GF_LOG_INFO, + "reconfigure called. setting priv->leader to %d\n", priv->leader); return 0; err: @@ -440,20 +441,25 @@ nsr_notify (xlator_t *this, int event, void *data, ...) index = nsr_get_child_index(this,data); if (index >= 0) { priv->kid_state |= (1 << index); + priv->up_children = nsr_count_up_kids(priv); gf_log (this->name, GF_LOG_INFO, "got CHILD_UP for %s, now %u kids", ((xlator_t *)data)->name, - nsr_count_up_kids(priv)); + priv->up_children); + if (priv->nsr_recon_start == _gf_true) { + nsr_recon_notify_event_add_child(priv, index); + } } break; case GF_EVENT_CHILD_DOWN: index = nsr_get_child_index(this,data); if (index >= 0) { priv->kid_state &= ~(1 << index); + priv->up_children = nsr_count_up_kids(priv); gf_log (this->name, GF_LOG_INFO, "got CHILD_DOWN for %s, now %u kids", ((xlator_t *)data)->name, - nsr_count_up_kids(priv)); + priv->up_children); } break; default: @@ -475,7 +481,7 @@ nsr_init (xlator_t *this) xlator_list_t *trav; pthread_t kid; uuid_t tmp_uuid; - char *my_name = NULL, *recon_file = NULL, *recon_pid_file = NULL, *ptr = NULL; + char *my_name = NULL, *morph_name = NULL, *recon_file = NULL, *recon_pid_file = NULL, *ptr = NULL; char *volname; extern xlator_t global_xlator; glusterfs_ctx_t *oldctx = global_xlator.ctx; @@ -552,34 +558,36 @@ nsr_init (xlator_t *this) gf_log (this->name, GF_LOG_ERROR, "vol name not generated. ???"); goto err; } - - recon_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("con") +1, gf_mt_nsr_private_t); - recon_pid_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("recon") +1, gf_mt_nsr_private_t); + + morph_name = GF_CALLOC (1, strlen(my_name) + 1, gf_mt_nsr_private_t); + strcpy(morph_name, my_name); + recon_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t); + recon_pid_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("recon") +1, gf_mt_nsr_private_t); if ((!recon_file) || (!recon_pid_file)) { gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name"); goto err; } - ptr = strchr (my_name, '/'); + ptr = strchr (morph_name, '/'); while (ptr) { *ptr = '-'; - ptr = strchr (my_name, '/'); + ptr = strchr (morph_name, '/'); } sprintf(recon_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_VOLUME_DIR_PREFIX, volname, GLUSTERD_BRICK_INFO_DIR); - strcat(recon_file, my_name); + strcat(recon_file, morph_name); strcat(recon_file, "-nsr-recon.vol"); sprintf(recon_pid_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_VOLUME_DIR_PREFIX, volname, "run"); - strcat(recon_pid_file, my_name); + strcat(recon_pid_file, morph_name); strcat(recon_pid_file, "-recon.pid"); - priv->vol_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("con") +1, gf_mt_nsr_private_t); + priv->vol_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t); if (!priv->vol_file) { gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name"); goto err; @@ -590,7 +598,7 @@ nsr_init (xlator_t *this) volname, GLUSTERD_BRICK_INFO_DIR); strcat(priv->vol_file, "con:"); - strcat(priv->vol_file, my_name); + strcat(priv->vol_file, morph_name); if (pthread_create(&kid,NULL,nsr_flush_thread,this) != 0) { gf_log (this->name, GF_LOG_ERROR, @@ -622,10 +630,17 @@ nsr_init (xlator_t *this) } + (void)pthread_create(&kid,NULL,nsr_recon_notify_thread,this); + while (priv->recon_notify_inited == 0) { + sleep(1); + } + (void)pthread_create(&kid,NULL,nsr_leader_thread,this); while (priv->leader_inited == 0) { sleep(1); } + + /* * Calling glfs_new changes old->ctx, even if THIS still points * to global_xlator. That causes problems later in the main -- cgit