/* Copyright (c) 2013 Red Hat, Inc. This file is part of GlusterFS. This file is licensed to you under your choice of the GNU Lesser General Public License, version 3 or any later version (LGPLv3 or later), or the GNU General Public License, version 2 (GPLv2), in all cases as published by the Free Software Foundation. */ #include //#include #include #ifndef _CONFIG_H #define _CONFIG_H #include "config.h" #endif #include "call-stub.h" #include "defaults.h" #include "xlator.h" #include "api/src/glfs.h" #include "api/src/glfs-internal.h" #ifndef NSR_SIM_ETCD #include "etcd-api.h" #endif #include "nsr-internal.h" #include "../../nsr-recon/src/recon_driver.h" #include "../../nsr-recon/src/recon_xlator.h" /* Vote format: UUID,vote_status,fitness,term_number */ #define VOTE_ELEMS 4 /* Whole match plus four actual pieces. */ #define DEFAULT_FITNESS 42 #define DEFAULT_KEY "nsr" #define LEADER_TTL 5 /* TBD: make this tunable */ typedef enum { LS_SUCCESS, LS_FAILURE, LS_ERROR } leader_retval_t; enum { NO_LEADER, TENTATIVE, CONFIRMED }; regex_t vote_re; // Simulation of etcd routines #ifndef NSR_SIM_ETCD #endif long nsr_get_fitness (xlator_t *this) { /* TBD: calculate based on presence/absence from terms */ return 42; } static void nsr_set_leader (xlator_t *this, etcd_session etcd) { long term = 0; etcd_result res; nsr_private_t *priv = this->private; char *term_key = priv->term_uuid; char n_t[sizeof(long)+1]; char *text = NULL; gf_log (this->name, GF_LOG_INFO, "Just became leader"); text = etcd_get(etcd, priv->term_uuid); if(text == NULL) { term = 0; } else { term = strtol(text, NULL, 10); } sprintf(n_t,"%ld",term+1); res = etcd_set(etcd, term_key,n_t,text,0); if(res != ETCD_OK) { gf_log (this->name, GF_LOG_ERROR, "failed to set term"); return; } priv->leader = _gf_true; if (priv->nsr_recon_start == _gf_false) { atomic_fetch_and(&(priv->fence_io), 0); return; } priv->current_term = term + 1; // Move this inside recon notify??? atomic_fetch_or(&(priv->fence_io), 1); nsr_recon_notify_event_set_leader(priv); return; } leader_retval_t nsr_get_leader (xlator_t *this, etcd_session etcd, char *key) { char *text = NULL; regmatch_t matches[VOTE_ELEMS]; char *nominee; long state; long fitness; char *vote = NULL; int retval = LS_ERROR; nsr_private_t *priv = this->private; for (;;sleep(1)) { if (text) { free(text); } text = etcd_get(etcd,key); if (text) { if (regexec(&vote_re,text,VOTE_ELEMS,matches,0) != 0) { gf_log (this->name, GF_LOG_ERROR, "got malformed vote %s\n", text); continue; } /* We can be destructive here, so convert commas. */ text[matches[1].rm_eo] = '\0'; text[matches[2].rm_eo] = '\0'; nominee = text + matches[1].rm_so; state = strtol(text+matches[2].rm_so,NULL,10); fitness = strtol(text+matches[3].rm_so,NULL,10); } else { nominee = NULL; state = NO_LEADER; fitness = 0; } if (state == CONFIRMED) { gf_log (this->name, GF_LOG_TRACE, "leader is %s\n",nominee); if (strcmp(nominee,priv->brick_uuid) == 0) { nsr_set_leader(this, etcd); retval = LS_SUCCESS; } else { priv->leader = _gf_false; retval = LS_FAILURE; } break; } /* TBD: override based on fitness */ if ((state >= TENTATIVE) && (strcmp(nominee, priv->brick_uuid) != 0)) { continue; } if (vote) { free(vote); } fitness = nsr_get_fitness(this); if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid, state+1,fitness) < 0) { gf_log (this->name, GF_LOG_ERROR, "failed to construct vote\n"); break; } if (text) { text[matches[1].rm_eo] = ','; text[matches[2].rm_eo] = ','; } if (etcd_set(etcd,key,vote,text,LEADER_TTL) != ETCD_OK) { gf_log (this->name, GF_LOG_ERROR, "failed to cast vote\n"); continue; } } if (text) { free(text); } if (vote) { free(vote); } return retval; } leader_retval_t nsr_confirm (xlator_t *this, etcd_session etcd, char *key) { char *vote; long fitness; nsr_private_t *priv = this->private; fitness = nsr_get_fitness(this); if (asprintf(&vote,"%s,%ld,%ld",priv->brick_uuid,(long)CONFIRMED, fitness) < 0) { fprintf (stderr, "%s: failed to construct confirmation\n", __func__); return LS_ERROR; } if (etcd_set(etcd,key,vote,vote,LEADER_TTL) != ETCD_OK) { fprintf (stderr, "%s: failed to confirm\n", __func__); free(vote); return LS_FAILURE; } free(vote); return LS_SUCCESS; } gf_boolean_t nsr_init_re (xlator_t *this) { static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; static int was_inited = 0; static char *vote_re_str = "([^,]+),([^,]+),([^,]+)"; gf_boolean_t retval = _gf_false; pthread_mutex_lock(&mutex); if (!was_inited) { if (regcomp(&vote_re,vote_re_str,REG_EXTENDED) == 0) { retval = _gf_true; } else { gf_log (this->name, GF_LOG_ERROR, "failed to set up vote regex\n"); } } pthread_mutex_unlock(&mutex); return retval; } void * nsr_leader_thread (void *arg) { xlator_t *this = (xlator_t *) arg; leader_retval_t retval; nsr_private_t *priv = this->private; if (!nsr_init_re(this)) { gf_log (this->name, GF_LOG_ERROR, "could not init regex"); return NULL; } gf_log (this->name, GF_LOG_INFO, "calling glfs_opens_str on servers %s", priv->etcd_servers); priv->etcd = etcd_open_str(priv->etcd_servers); if (!(priv->etcd)) { gf_log (this->name, GF_LOG_ERROR, "failed to open etcd session\n"); return NULL; } priv->leader_inited = 1; for (;;) { if (nsr_get_leader(this,priv->etcd,priv->vol_uuid) == LS_ERROR) { break; } if (priv->leader) { do { sleep(1); retval = nsr_confirm(this,priv->etcd,priv->vol_uuid); } while (retval == LS_SUCCESS); if (retval == LS_ERROR) { break; } } else { sleep(1); } } etcd_close_str(priv->etcd); return NULL; }