Merge "nsr: change leader-election code to use new etcd_lock"

2 files changed, 69 insertions, 182 deletions

diff --git a/xlators/cluster/nsr-server/src/etcd-sim.c b/xlators/cluster/nsr-server/src/etcd-sim.c
index 782b5f617..d0bea12c7 100644
--- a/xlators/cluster/nsr-server/src/etcd-sim.c
+++ b/xlators/cluster/nsr-server/src/etcd-sim.c
@@ -34,6 +34,8 @@
 #include <fcntl.h>
 #include <sys/file.h>
 
+#include "mem-pool.h"
+
 /*
  * Mock implementation of etcd
  * The etcd file is simulated in /tmp/<server-names>
@@ -244,3 +246,35 @@ etcd_watch (etcd_session this, char *pfx, char **keyp, char **valuep,
 {
 	return ETCD_WTF;
 }
+
+etcd_result
+etcd_lock (etcd_session session_as_void, char *key, unsigned int ttl,
+           char *index_in, char **index_out)
+{
+        char    *path;
+        int     fd;
+
+        if (!index_in) {
+                if (gf_asprintf(&path,"/var/tmp/%s",key) < 0) {
+                        return ETCD_WTF;
+                }
+                fd = open(path,O_RDWR|O_CREAT,0666);
+                GF_FREE(path);
+                if (fd < 0) {
+                        return ETCD_WTF;
+                }
+                if (flock(fd,LOCK_EX) < 0) {
+                        close(fd);
+                        return ETCD_WTF;
+                }
+                *index_out = strdup("42");
+        }
+
+        /*
+         * Yes, we leak an fd by not closing it here (and nobody else even
+         * knows about it).  That would be awful in any other context, but
+         * for test scripts it won't matter.
+         */
+        return ETCD_OK;
+}
+
diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c
index 6c3ca6d4f..73b9f66d5 100644
--- a/xlators/cluster/nsr-server/src/leader.c
+++ b/xlators/cluster/nsr-server/src/leader.c
@@ -30,28 +30,8 @@
 #include "../../nsr-recon/src/recon_driver.h"
 #include "../../nsr-recon/src/recon_xlator.h"
 
-
-/* Vote format: UUID,vote_status,fitness,term_number */
-#define VOTE_ELEMS      4       /* Whole match plus four actual pieces. */
-#define DEFAULT_FITNESS 42
-#define DEFAULT_KEY     "nsr"
-#define LEADER_TTL      5       /* TBD: make this tunable */
-
-typedef enum { LS_SUCCESS, LS_FAILURE, LS_ERROR } leader_retval_t;
-enum { NO_LEADER, TENTATIVE, CONFIRMED };
-
-regex_t vote_re;
-
-// Simulation of etcd routines
-#ifndef NSR_SIM_ETCD
-#endif
-
-long
-nsr_get_fitness (xlator_t *this)
-{
-        /* TBD: calculate based on presence/absence from terms */
-        return 42;
-}
+#define NSR_KEY         "xyzzy"
+#define NSR_TTL         5
 
 static void
 nsr_set_leader (xlator_t *this, etcd_session etcd)
@@ -94,161 +74,17 @@ nsr_set_leader (xlator_t *this, etcd_session etcd)
         return;
 }
 
-
-leader_retval_t
-nsr_get_leader (xlator_t *this, etcd_session etcd, char *key)
-{
-        char            *text   = NULL;
-        regmatch_t      matches[VOTE_ELEMS];
-        char            *nominee;
-        long            state;
-        long            fitness;
-        char            *vote   = NULL;
-        int             retval  = LS_ERROR;
-        nsr_private_t   *priv   = this->private;
-
-        for (;;sleep(1)) {
-                
-                if (text) {
-                        free(text);
-                }
-                
-                text = etcd_get(etcd,key);
-                if (text) {
-                        if (regexec(&vote_re,text,VOTE_ELEMS,matches,0) != 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "got malformed vote %s\n", text);
-                                continue;
-                        }
-                        /* We can be destructive here, so convert commas. */
-                        text[matches[1].rm_eo] = '\0';
-                        text[matches[2].rm_eo] = '\0';
-                        nominee = text + matches[1].rm_so;
-                        state = strtol(text+matches[2].rm_so,NULL,10);
-                        fitness = strtol(text+matches[3].rm_so,NULL,10);
-                }
-                else {
-                        nominee = NULL;
-                        state = NO_LEADER;
-                        fitness = 0;
-                }
-
-                if (state == CONFIRMED) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "leader is %s\n",nominee);
-                        if (strcmp(nominee,priv->brick_uuid) == 0) {
-                                nsr_set_leader(this, etcd);
-                                retval = LS_SUCCESS;
-                        }
-                        else {
-                                priv->leader = _gf_false;
-                                retval = LS_FAILURE;
-                        }
-                        break;
-                }
-
-                /* TBD: override based on fitness */
-                if ((state >= TENTATIVE) && (strcmp(nominee,
-                                                    priv->brick_uuid) != 0)) {
-                        continue;
-                }
-
-                if (vote) {
-                        free(vote);
-                }
-
-                fitness = nsr_get_fitness(this);
-                if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid,
-                                                state+1,fitness) < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to construct vote\n");
-                        break;
-                }
-
-                if (text) {
-                        text[matches[1].rm_eo] = ',';
-                        text[matches[2].rm_eo] = ',';
-                }
-                if (etcd_set(etcd,key,vote,text,LEADER_TTL) != ETCD_OK) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to cast vote\n");
-                        continue;
-                }
-                
-        }
-
-        if (text) {
-                free(text);
-        }
-        if (vote) {
-                free(vote);
-        }
-        return retval;
-}
-
-leader_retval_t
-nsr_confirm (xlator_t *this, etcd_session etcd, char *key)
-{
-        char            *vote;
-        long            fitness;
-        nsr_private_t   *priv   = this->private;
-
-        fitness = nsr_get_fitness(this);
-        if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid,(long)CONFIRMED,
-                                                         fitness) < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to construct confirmation");
-                return LS_ERROR;
-        }
-
-        if (etcd_set(etcd,key,vote,vote,LEADER_TTL) != ETCD_OK) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to confirm");
-                free(vote);
-                return LS_FAILURE;
-        }
-        
-        free(vote);
-        return LS_SUCCESS;
-}
-
-gf_boolean_t
-nsr_init_re (xlator_t *this)
-{
-        static pthread_mutex_t  mutex           = PTHREAD_MUTEX_INITIALIZER;
-        static int              was_inited      = 0;
-        static char             *vote_re_str    = "([^,]+),([^,]+),([^,]+)";
-        gf_boolean_t            retval          = _gf_false;
-
-        pthread_mutex_lock(&mutex);
-        if (!was_inited) {
-                if (regcomp(&vote_re,vote_re_str,REG_EXTENDED) == 0) {
-                        retval = _gf_true;
-                }
-                else {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set up vote regex\n");
-                }
-        }
-        pthread_mutex_unlock(&mutex);
-
-        return retval;
-}
-
-
 void *
 nsr_leader_thread (void *arg)
 {
-        xlator_t *this = (xlator_t *) arg;
-        leader_retval_t retval;
-        nsr_private_t   *priv   = this->private;
-
-        if (!nsr_init_re(this)) {
-                gf_log (this->name, GF_LOG_ERROR, "could not init regex");
-                return NULL;
-        }
+        xlator_t        *this           = (xlator_t *) arg;
+        nsr_private_t   *priv           = this->private;
+        etcd_result     res;
+        char            *index_in       = NULL;
+        char            *index_out      = NULL;
 
         gf_log (this->name, GF_LOG_INFO,
-                "calling glfs_opens_str on servers %s", priv->etcd_servers);
+                "calling glfs_open_str on servers %s", priv->etcd_servers);
 
         priv->etcd = etcd_open_str(priv->etcd_servers);
         if (!(priv->etcd)) {
@@ -260,19 +96,36 @@ nsr_leader_thread (void *arg)
         priv->leader_inited = 1;
 
         for (;;) {
-                if (nsr_get_leader(this,priv->etcd,priv->vol_uuid) == LS_ERROR) {
-                        break;
-                }
-                if (priv->leader) {
-                        do {
-                                sleep(1);
-                                retval = nsr_confirm(this,priv->etcd,priv->vol_uuid);
-                        } while (retval == LS_SUCCESS);
-                        if (retval == LS_ERROR) {
+                /* Not leader yet.  Try to become leader. */
+                for (;;) {
+                        res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+                                         index_in, &index_out);
+                        if (res == ETCD_OK) {
                                 break;
                         }
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "etcd_lock failed (%d)", res);
+                        sleep(1);
                 }
-                else {
+                /* We're there.  Notify other parts of the code. */
+                nsr_set_leader(this,priv->etcd);
+                /* Try to retain leadership. */
+                for (;;) {
+                        index_in = index_out;
+                        index_out = NULL;
+                        res = etcd_lock (priv->etcd, NSR_KEY, NSR_TTL,
+                                         index_in, &index_out);
+                        if (index_in && (index_in != index_out)) {
+                                free(index_in);
+                        }
+                        if (res != ETCD_OK) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "lost leadership (%d)", res);
+                                if (index_out) {
+                                        free(index_out);
+                                }
+                                break;
+                        }
                         sleep(1);
                 }
         }