From 63f9e2a49573a0b539c0082fd0c08c1b4d4db983 Mon Sep 17 00:00:00 2001 From: Pavan Sondur Date: Sun, 22 Aug 2010 14:08:43 +0000 Subject: cluster/afr: Use 2 phase locking for transactions and self heal. Signed-off-by: Pavan Vilas Sondur Signed-off-by: Anand V. Avati BUG: 960 () URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=960 --- xlators/cluster/afr/src/afr-self-heal-data.c | 203 ++++++++++----------------- 1 file changed, 73 insertions(+), 130 deletions(-) (limited to 'xlators/cluster/afr/src/afr-self-heal-data.c') diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c index 366cac81721..8e87bc55545 100644 --- a/xlators/cluster/afr/src/afr-self-heal-data.c +++ b/xlators/cluster/afr/src/afr-self-heal-data.c @@ -56,7 +56,6 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this) afr_local_t *local = NULL; afr_self_heal_t *sh = NULL; afr_private_t *priv = NULL; - int i = 0; local = frame->local; sh = &local->self_heal; @@ -73,8 +72,8 @@ afr_sh_data_done (call_frame_t *frame, xlator_t *this) sh->healing_fd = NULL; } - for (i = 0; i < priv->child_count; i++) - sh->locked_nodes[i] = 0; +/* for (i = 0; i < priv->child_count; i++) */ +/* sh->locked_nodes[i] = 0; */ gf_log (this->name, GF_LOG_TRACE, "self heal of %s completed", @@ -268,58 +267,18 @@ afr_sh_data_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int afr_sh_data_unlock (call_frame_t *frame, xlator_t *this) { - struct flock flock; - int i = 0; - int call_count = 0; + afr_local_t *local = NULL; + afr_internal_lock_t *int_lock = NULL; + afr_self_heal_t *sh = NULL; - afr_local_t * local = NULL; - afr_private_t * priv = NULL; - afr_self_heal_t * sh = NULL; - - - local = frame->local; - sh = &local->self_heal; - priv = this->private; - - if (sh->data_lock_held) { - /* not our job to unlock, proceed to close */ - - afr_sh_data_close (frame, this); - return 0; - } + local = frame->local; + int_lock = &local->internal_lock; + sh = &local->self_heal; - for (i = 0; i < priv->child_count; i++) { - if (sh->locked_nodes[i]) - call_count++; - } + GF_ASSERT (!sh->data_lock_held); - if (call_count == 0) { - afr_sh_data_close (frame, this); - return 0; - } - - local->call_count = call_count; - - flock.l_start = 0; - flock.l_len = 0; - flock.l_type = F_UNLCK; - - for (i = 0; i < priv->child_count; i++) { - if (sh->locked_nodes[i]) { - gf_log (this->name, GF_LOG_TRACE, - "unlocking %s on subvolume %s", - local->loc.path, priv->children[i]->name); - - STACK_WIND_COOKIE (frame, afr_sh_data_unlck_cbk, - (void *) (long) i, - priv->children[i], - priv->children[i]->fops->inodelk, - this->name, - &local->loc, F_SETLK, &flock); - if (!--call_count) - break; - } - } + int_lock->lock_cbk = afr_sh_data_close; + afr_unlock (frame, this); return 0; } @@ -329,13 +288,18 @@ int afr_sh_data_finish (call_frame_t *frame, xlator_t *this) { afr_local_t *local = NULL; + afr_self_heal_t *sh = NULL; local = frame->local; + sh = &local->self_heal; gf_log (this->name, GF_LOG_TRACE, "finishing data selfheal of %s", local->loc.path); - afr_sh_data_unlock (frame, this); + if (!sh->data_lock_held) + afr_sh_data_unlock (frame, this); + else + afr_sh_data_close (frame, this); return 0; } @@ -388,7 +352,7 @@ afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this) priv->child_count, AFR_DATA_TRANSACTION); erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count, - gf_afr_mt_dict_t); + gf_afr_mt_dict_t); for (i = 0; i < priv->child_count; i++) { if (sh->xattr[i]) { @@ -772,8 +736,8 @@ afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr) gf_afr_mt_int32_t); for (i = 0; i < priv->child_count; i++) { sh->pending_matrix[i] = GF_CALLOC (sizeof (int32_t), - priv->child_count, - gf_afr_mt_int32_t); + priv->child_count, + gf_afr_mt_int32_t); } sh->sources = GF_CALLOC (priv->child_count, sizeof (*sh->sources), @@ -958,96 +922,79 @@ afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this) int -afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, int child_index); +afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this); int -afr_sh_data_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this, - int32_t op_ret, int32_t op_errno) +afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this) { - afr_local_t *local = NULL; - afr_self_heal_t *sh = NULL; - int child_index = (long) cookie; - - /* TODO: what if lock fails? */ - - local = frame->local; - sh = &local->self_heal; + afr_internal_lock_t *int_lock = NULL; + afr_local_t *local = NULL; - LOCK (&frame->lock); - { - if (op_ret == -1) { - sh->locked_nodes[child_index] = 0; + local = frame->local; + int_lock = &local->internal_lock; - gf_log (this->name, GF_LOG_DEBUG, - "locking of %s on child %d failed: %s", - local->loc.path, child_index, - strerror (op_errno)); - } else { - sh->locked_nodes[child_index] = 1; - sh->lock_count++; - - gf_log (this->name, GF_LOG_TRACE, - "inode of %s on child %d locked", - local->loc.path, child_index); - } - } - UNLOCK (&frame->lock); + if (int_lock->lock_op_ret < 0) { + gf_log (this->name, GF_LOG_DEBUG, + "Blocking inodelks failed."); + afr_sh_data_done (frame, this); + } else { - afr_sh_data_lock_rec (frame, this, child_index + 1); + gf_log (this->name, GF_LOG_DEBUG, + "Blocking inodelks done. Proceeding to FOP"); + afr_sh_data_fxattrop (frame, this); + } - return 0; + return 0; } - int -afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, int child_index) +afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this) { - struct flock flock; - int i = 0; + afr_internal_lock_t *int_lock = NULL; + afr_local_t *local = NULL; - afr_local_t * local = NULL; - afr_private_t * priv = NULL; - afr_self_heal_t * sh = NULL; + local = frame->local; + int_lock = &local->internal_lock; - local = frame->local; - sh = &local->self_heal; - priv = this->private; + /* Initiate blocking locks if non-blocking has failed */ + if (int_lock->lock_op_ret < 0) { + gf_log (this->name, GF_LOG_DEBUG, + "Non blocking inodelks failed. Proceeding to blocking"); + int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk; + afr_blocking_lock (frame, this); + } else { - flock.l_start = 0; - flock.l_len = 0; - flock.l_type = F_WRLCK; + gf_log (this->name, GF_LOG_DEBUG, + "Non blocking inodelks done. Proceeding to FOP"); + afr_sh_data_fxattrop (frame, this); + } - /* skip over children that are down */ - while ((child_index < priv->child_count) - && !local->child_up[child_index]) - child_index++; + return 0; +} - if ((child_index == priv->child_count) && - sh->lock_count == 0) { +int +afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this) +{ + afr_internal_lock_t *int_lock = NULL; + afr_local_t *local = NULL; + afr_self_heal_t *sh = NULL; - gf_log (this->name, GF_LOG_DEBUG, - "unable to lock on even one child"); + local = frame->local; + int_lock = &local->internal_lock; + sh = &local->self_heal; - afr_sh_data_done (frame, this); - return 0; - } + int_lock->transaction_lk_type = AFR_SELFHEAL_LK; + int_lock->selfheal_lk_type = AFR_DATA_SELF_HEAL_LK; - if ((child_index == priv->child_count) - || (sh->lock_count == afr_lock_server_count (priv, AFR_DATA_TRANSACTION))) { - afr_sh_data_fxattrop (frame, this); - return 0; - } + afr_set_lock_number (frame, this); + + int_lock->lk_flock.l_start = 0; + int_lock->lk_flock.l_len = 0; + int_lock->lk_flock.l_type = F_WRLCK; + int_lock->lock_cbk = afr_sh_data_post_nonblocking_inodelk_cbk; - gf_log (this->name, GF_LOG_TRACE, - "locking %s on subvolume %s", - local->loc.path, priv->children[i]->name); + afr_nonblocking_inodelk (frame, this); - STACK_WIND_COOKIE (frame, afr_sh_data_lock_cbk, - (void *) (long) child_index, - priv->children[i], - priv->children[i]->fops->inodelk, - this->name, - &local->loc, F_SETLKW, &flock); return 0; } @@ -1060,7 +1007,6 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this) afr_private_t * priv = NULL; afr_self_heal_t * sh = NULL; - int i = 0; local = frame->local; sh = &local->self_heal; @@ -1074,10 +1020,7 @@ afr_sh_data_lock (call_frame_t *frame, xlator_t *this) return 0; } - for (i = 0; i < priv->child_count; i++) - sh->locked_nodes[i] = 0; - - return afr_sh_data_lock_rec (frame, this, 0); + return afr_sh_data_lock_rec (frame, this); } -- cgit