Changes to NSR reconciliation code.

Following is list of changes:
1) Simulation of etcd using a file as a registry protected using locks.
2) Implement notifications for child up and child down.
3) Join a new brick into quorum.
4) add support for proper fencing and draining of IO required for reconciliaiton
5) misc changes and addressed review comments.

Change-Id: Iddd1137ad6205252ed03301888bb1e83fa2221e0
Signed-off-by: Raghavan P <rpichai@redhat.com>

1 files changed, 30 insertions, 15 deletions

diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c
index 3707b3003..f85368456 100644
--- a/xlators/cluster/nsr-server/src/nsr.c
+++ b/xlators/cluster/nsr-server/src/nsr.c
@@ -258,7 +258,6 @@ nsr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc,
                       const char *name, dict_t *xdata)
 {
         dict_t          *result;
-        uint8_t         up;
         nsr_private_t   *priv   = this->private;
 
         if (!priv->leader) {
@@ -279,8 +278,8 @@ nsr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 goto dn_failed;
         }
 
-        up = nsr_count_up_kids(this->private);
-        if (dict_set_uint32(result,NSR_REP_COUNT_XATTR,up) != 0) {
+        priv->up_children = nsr_count_up_kids(this->private);
+        if (dict_set_uint32(result,NSR_REP_COUNT_XATTR,priv->up_children) != 0) {
                 goto dsu_failed;
         }
 
@@ -399,6 +398,8 @@ nsr_reconfigure (xlator_t *this, dict_t *options)
         nsr_private_t   *priv   = this->private;
 
 	GF_OPTION_RECONF ("leader", priv->leader, options, bool, err);
+        gf_log (this->name, GF_LOG_INFO,
+                "reconfigure called. setting priv->leader to %d\n", priv->leader);
         return 0;
 
 err:
@@ -440,20 +441,25 @@ nsr_notify (xlator_t *this, int event, void *data, ...)
                 index = nsr_get_child_index(this,data);
                 if (index >= 0) {
                         priv->kid_state |= (1 << index);
+                        priv->up_children = nsr_count_up_kids(priv);
                         gf_log (this->name, GF_LOG_INFO,
                                 "got CHILD_UP for %s, now %u kids",
                                 ((xlator_t *)data)->name,
-                                nsr_count_up_kids(priv));
+                                priv->up_children);
+                        if (priv->nsr_recon_start == _gf_true) {
+                                nsr_recon_notify_event_add_child(priv, index);
+                        }
                 }
                 break;
         case GF_EVENT_CHILD_DOWN:
                 index = nsr_get_child_index(this,data);
                 if (index >= 0) {
                         priv->kid_state &= ~(1 << index);
+                        priv->up_children = nsr_count_up_kids(priv);
                         gf_log (this->name, GF_LOG_INFO,
                                 "got CHILD_DOWN for %s, now %u kids",
                                 ((xlator_t *)data)->name,
-                                nsr_count_up_kids(priv));
+                                priv->up_children);
                 }
                 break;
         default:
@@ -475,7 +481,7 @@ nsr_init (xlator_t *this)
         xlator_list_t   *trav;
         pthread_t       kid;
         uuid_t          tmp_uuid;
-        char            *my_name = NULL, *recon_file = NULL, *recon_pid_file = NULL, *ptr = NULL;
+        char            *my_name = NULL, *morph_name = NULL, *recon_file = NULL, *recon_pid_file = NULL, *ptr = NULL;
         char *volname;
         extern xlator_t global_xlator;
         glusterfs_ctx_t *oldctx = global_xlator.ctx;
@@ -552,34 +558,36 @@ nsr_init (xlator_t *this)
                 gf_log (this->name, GF_LOG_ERROR, "vol name not generated. ???");
                 goto err;
         }
-
-        recon_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("con") +1, gf_mt_nsr_private_t);
-        recon_pid_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("recon") +1, gf_mt_nsr_private_t);
+        
+        morph_name = GF_CALLOC (1, strlen(my_name) + 1, gf_mt_nsr_private_t);
+        strcpy(morph_name, my_name);
+        recon_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t);
+        recon_pid_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("recon") +1, gf_mt_nsr_private_t);
         if ((!recon_file) || (!recon_pid_file)) {
                 gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name");
                 goto err;
         }
-        ptr = strchr (my_name, '/');
+        ptr = strchr (morph_name, '/');
         while (ptr) {
                 *ptr = '-';
-                ptr = strchr (my_name, '/');
+                ptr = strchr (morph_name, '/');
         }
 
         sprintf(recon_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR,
                                      GLUSTERD_VOLUME_DIR_PREFIX,
                                      volname,
                                      GLUSTERD_BRICK_INFO_DIR);
-        strcat(recon_file, my_name);
+        strcat(recon_file, morph_name);
         strcat(recon_file, "-nsr-recon.vol");
 
         sprintf(recon_pid_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR,
                                      GLUSTERD_VOLUME_DIR_PREFIX,
                                      volname,
                                      "run");
-        strcat(recon_pid_file, my_name);
+        strcat(recon_pid_file, morph_name);
         strcat(recon_pid_file, "-recon.pid");
 
-        priv->vol_file = GF_CALLOC (1,PATH_MAX + strlen(my_name) + strlen("con") +1, gf_mt_nsr_private_t);
+        priv->vol_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t);
         if (!priv->vol_file) {
                 gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name");
                 goto err;
@@ -590,7 +598,7 @@ nsr_init (xlator_t *this)
                 volname,
                 GLUSTERD_BRICK_INFO_DIR);
         strcat(priv->vol_file, "con:");
-        strcat(priv->vol_file, my_name);
+        strcat(priv->vol_file, morph_name);
 
         if (pthread_create(&kid,NULL,nsr_flush_thread,this) != 0) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -622,10 +630,17 @@ nsr_init (xlator_t *this)
         }
 
 
+        (void)pthread_create(&kid,NULL,nsr_recon_notify_thread,this);
+        while (priv->recon_notify_inited == 0) {
+                  sleep(1);
+        }
+
         (void)pthread_create(&kid,NULL,nsr_leader_thread,this);
         while (priv->leader_inited == 0) {
                   sleep(1);
         }
+
+
         /*
          * Calling glfs_new changes old->ctx, even if THIS still points
          * to global_xlator.  That causes problems later in the main