From 614904fa7a31bf6f69074238b7e710a20e05e1bb Mon Sep 17 00:00:00 2001 From: Gaurav Yadav Date: Thu, 5 Oct 2017 23:44:46 +0530 Subject: glusterd : introduce timer in mgmt_v3_lock Problem: In a multinode environment, if two of the op-sm transactions are initiated on one of the receiver nodes at the same time, there might be a possibility that glusterd may end up in stale lock. Solution: During mgmt_v3_lock a registration is made to gf_timer_call_after which release the lock after certain period of time Change-Id: I16cc2e5186a2e8a5e35eca2468b031811e093843 BUG: 1499004 Signed-off-by: Gaurav Yadav --- xlators/mgmt/glusterd/src/glusterd-locks.c | 220 +++++++++++++++++++++++++++-- 1 file changed, 206 insertions(+), 14 deletions(-) (limited to 'xlators/mgmt/glusterd/src/glusterd-locks.c') diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c index 146092db79b..c7951b3cf24 100644 --- a/xlators/mgmt/glusterd/src/glusterd-locks.c +++ b/xlators/mgmt/glusterd/src/glusterd-locks.c @@ -94,6 +94,50 @@ glusterd_mgmt_v3_lock_fini () dict_unref (priv->mgmt_v3_lock); } +/* Initialize the global mgmt_v3_timer lock list(dict) when + * glusterd is spawned */ +int32_t +glusterd_mgmt_v3_lock_timer_init () +{ + int32_t ret = -1; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO ("glusterd", this, out); + + priv = this->private; + GF_VALIDATE_OR_GOTO (this->name, priv, out); + + priv->mgmt_v3_lock_timer = dict_new (); + if (!priv->mgmt_v3_lock_timer) + goto out; + + ret = 0; +out: + return ret; +} + +/* Destroy the global mgmt_v3_timer lock list(dict) when + * glusterd cleanup is performed */ +void +glusterd_mgmt_v3_lock_timer_fini () +{ + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO ("glusterd", this, out); + + priv = this->private; + GF_VALIDATE_OR_GOTO (this->name, priv, out); + + if (priv->mgmt_v3_lock_timer) + dict_unref (priv->mgmt_v3_lock_timer); +out: + return; +} + int32_t glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid) { @@ -513,17 +557,23 @@ int32_t glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, uint32_t *op_errno, char *type) { - char key[PATH_MAX] = ""; - int32_t ret = -1; - glusterd_mgmt_v3_lock_obj *lock_obj = NULL; - glusterd_conf_t *priv = NULL; - gf_boolean_t is_valid = _gf_true; - uuid_t owner = {0}; - xlator_t *this = NULL; - char *bt = NULL; + char key[PATH_MAX] = ""; + int32_t ret = -1; + glusterd_mgmt_v3_lock_obj *lock_obj = NULL; + glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL; + glusterd_conf_t *priv = NULL; + gf_boolean_t is_valid = _gf_true; + uuid_t owner = {0}; + xlator_t *this = NULL; + char *bt = NULL; + struct timespec delay = {0}; + char *key_dup = NULL; + glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL; + xlator_t *mgmt_lock_timer_xl = NULL; this = THIS; GF_ASSERT (this); + priv = this->private; GF_ASSERT (priv); @@ -594,6 +644,42 @@ glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, uint32_t *op_errno, goto out; } + mgmt_lock_timer = GF_CALLOC (1, sizeof(glusterd_mgmt_v3_lock_timer), + gf_common_mt_mgmt_v3_lock_timer_t); + + if (!mgmt_lock_timer) { + ret = -1; + goto out; + } + + mgmt_lock_timer->xl = THIS; + key_dup = gf_strdup (key); + delay.tv_sec = priv->mgmt_v3_lock_timeout; + delay.tv_nsec = 0; + + ret = -1; + mgmt_lock_timer_xl = mgmt_lock_timer->xl; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl, out); + + mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx, out); + + mgmt_lock_timer->timer = gf_timer_call_after + (mgmt_lock_timer_ctx, delay, + gd_mgmt_v3_unlock_timer_cbk, + key_dup); + + ret = dict_set_bin (priv->mgmt_v3_lock_timer, key, mgmt_lock_timer, + sizeof (glusterd_mgmt_v3_lock_timer)); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Unable to set timer in mgmt_v3 lock"); + GF_FREE (mgmt_lock_timer); + goto out; + } + + /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/ if ((bt = gf_backtrace_save (NULL))) { snprintf (key, sizeof (key), "debug.last-success-bt-%s-%s", @@ -617,18 +703,99 @@ out: return ret; } +/* + * This call back will ensure to unlock the lock_obj, in case we hit a situation + * where unlocking failed and stale lock exist*/ +void +gd_mgmt_v3_unlock_timer_cbk (void *data) +{ + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL; + char *key = NULL; + char *type = NULL; + char bt_key[PATH_MAX] = ""; + char name[PATH_MAX] = ""; + int32_t ret = -1; + glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL; + xlator_t *mgmt_lock_timer_xl = NULL; + + this = THIS; + GF_VALIDATE_OR_GOTO ("glusterd", this, out); + + conf = this->private; + GF_VALIDATE_OR_GOTO (this->name, conf, out); + + gf_log (THIS->name, GF_LOG_INFO, "In gd_mgmt_v3_unlock_timer_cbk"); + GF_ASSERT (NULL != data); + key = (char *)data; + + dict_del (conf->mgmt_v3_lock, key); + + type = strrchr (key, '_'); + strncpy (name, key, strlen (key) - strlen (type) - 1); + + ret = snprintf (bt_key, PATH_MAX, "debug.last-success-bt-%s-%s", + name, type + 1); + if (ret != strlen ("debug.last-success-bt-") + strlen (name) + + strlen (type)) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_CREATE_KEY_FAIL, "Unable to create backtrace " + "key"); + goto out; + } + + dict_del (conf->mgmt_v3_lock, bt_key); + + ret = dict_get_bin (conf->mgmt_v3_lock_timer, key, + (void **)&mgmt_lock_timer); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Unable to get lock owner in mgmt_v3 lock"); + goto out; + } + +out: + if (mgmt_lock_timer->timer) { + mgmt_lock_timer_xl = mgmt_lock_timer->xl; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl, + ret_function); + + mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx, + ret_function); + + gf_timer_call_cancel (mgmt_lock_timer_ctx, + mgmt_lock_timer->timer); + GF_FREE(key); + dict_del (conf->mgmt_v3_lock_timer, bt_key); + mgmt_lock_timer->timer = NULL; + } + +ret_function: + + return; + +} + int32_t glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type) { - char key[PATH_MAX] = ""; - int32_t ret = -1; - gf_boolean_t is_valid = _gf_true; - glusterd_conf_t *priv = NULL; - uuid_t owner = {0}; - xlator_t *this = NULL; + char key[PATH_MAX] = ""; + char key_dup[PATH_MAX] = ""; + int32_t ret = -1; + gf_boolean_t is_valid = _gf_true; + glusterd_conf_t *priv = NULL; + glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL; + uuid_t owner = {0}; + xlator_t *this = NULL; + glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL; + xlator_t *mgmt_lock_timer_xl = NULL; this = THIS; GF_ASSERT (this); + priv = this->private; GF_ASSERT (priv); @@ -657,6 +824,7 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type) ret = -1; goto out; } + strncpy (key_dup, key, strlen(key)); gf_msg_debug (this->name, 0, "Trying to release lock of %s %s for %s as %s", @@ -690,6 +858,15 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type) /* Removing the mgmt_v3 lock from the global list */ dict_del (priv->mgmt_v3_lock, key); + ret = dict_get_bin (priv->mgmt_v3_lock_timer, key, + (void **)&mgmt_lock_timer); + if (ret) { + gf_msg (this->name, GF_LOG_ERROR, 0, + GD_MSG_DICT_SET_FAILED, + "Unable to get mgmt lock key in mgmt_v3 lock"); + goto out; + } + /* Remove the backtrace key as well */ ret = snprintf (key, sizeof(key), "debug.last-success-bt-%s-%s", name, type); @@ -708,7 +885,22 @@ glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type) type, name); ret = 0; + /* Release owner refernce which was held during lock */ + if (mgmt_lock_timer->timer) { + ret = -1; + mgmt_lock_timer_xl = mgmt_lock_timer->xl; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_xl, out); + + mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx; + GF_VALIDATE_OR_GOTO (this->name, mgmt_lock_timer_ctx, out); + ret = 0; + gf_timer_call_cancel (mgmt_lock_timer_ctx, + mgmt_lock_timer->timer); + dict_del (priv->mgmt_v3_lock_timer, key_dup); + mgmt_lock_timer->timer = NULL; + } out: + gf_msg_trace (this->name, 0, "Returning %d", ret); return ret; } -- cgit