summaryrefslogtreecommitdiffstats
path: root/xlators/cluster/afr/src/afr-transaction.c
diff options
context:
space:
mode:
authorRavishankar N <ravishankar@redhat.com>2015-04-15 22:22:08 +0530
committerVijay Bellur <vbellur@redhat.com>2015-05-05 00:06:34 -0700
commit25e8e74eb7b81ccd114a9833371a3f72d284c48d (patch)
tree0c82b2ad9133c55c74852ce65d027bc36ed6cac5 /xlators/cluster/afr/src/afr-transaction.c
parent6d7428d2018c061ca2791443bd90980f9755ded3 (diff)
afr: add arbitration support
Add logic in afr to work in conjunction with the arbiter xlator when a replica 3 arbiter volume is created. More specifically, this patch: * Enables full locks for afr data transaction for such volumes. * Removes the upfront marking of pending xattrs at the time of pre-op and defer it to post-op. (This is an arbiter independent change and is made for all afr transactions.) * After pre-op stage, check if we can proceed with the fop stage without ending up in split-brain by examining the changelog xattrs. * Unwinds the fop with failure if only one source was available at the time of pre-op and the fop happened to fail on particular source brick. * Skips data self-heal if arbiter brick is the only source available. * Adds the arbiter-count option to the shd graph. This patch is a part of the arbiter logic implementation for 3 way AFR details of which can be found at http://review.gluster.org/#/c/9656/ Change-Id: I9603db9d04de5626eb2f4d8d959ef5b46113561d BUG: 1199985 Signed-off-by: Ravishankar N <ravishankar@redhat.com> Reviewed-on: http://review.gluster.org/10258 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Diffstat (limited to 'xlators/cluster/afr/src/afr-transaction.c')
-rw-r--r--xlators/cluster/afr/src/afr-transaction.c183
1 files changed, 152 insertions, 31 deletions
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index d2111060035..a2023884465 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -15,7 +15,7 @@
#include "afr.h"
#include "afr-transaction.h"
-
+#include "afr-self-heal.h"
#include <signal.h>
gf_boolean_t
@@ -139,14 +139,130 @@ __mark_all_success (call_frame_t *frame, xlator_t *this)
}
}
+void
+afr_compute_pre_op_sources (call_frame_t *frame, xlator_t *this)
+{
+ afr_local_t *local = NULL;
+ afr_private_t *priv = NULL;
+ afr_transaction_type type = -1;
+ dict_t *xdata = NULL;
+ int **matrix = NULL;
+ int idx = -1;
+ int i = 0;
+ int j = 0;
+
+ priv = this->private;
+ local = frame->local;
+ type = local->transaction.type;
+ idx = afr_index_for_transaction_type (type);
+ matrix = ALLOC_MATRIX (priv->child_count, int);
+
+ for (i = 0; i < priv->child_count; i++) {
+ if (!local->transaction.pre_op_xdata[i])
+ continue;
+ xdata = local->transaction.pre_op_xdata[i];
+ afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
+ }
+
+ memset (local->transaction.pre_op_sources, 1, priv->child_count);
+
+ /*If lock or pre-op failed on a brick, it is not a source. */
+ for (i = 0; i < priv->child_count; i++) {
+ if (local->transaction.failed_subvols[i])
+ local->transaction.pre_op_sources[i] = 0;
+ }
+
+ /* If brick is blamed by others, it is not a source. */
+ for (i = 0; i < priv->child_count; i++)
+ for (j = 0; j < priv->child_count; j++)
+ if (matrix[i][j] != 0)
+ local->transaction.pre_op_sources[j] = 0;
+
+ /*We don't need the xattrs any more. */
+ for (i = 0; i < priv->child_count; i++)
+ if (local->transaction.pre_op_xdata[i]) {
+ dict_unref (local->transaction.pre_op_xdata[i]);
+ local->transaction.pre_op_xdata[i] = NULL;
+ }
+}
+
+void
+afr_txn_arbitrate_fop_cbk (call_frame_t *frame, xlator_t *this)
+{
+ afr_local_t *local = NULL;
+ afr_private_t *priv = NULL;
+ gf_boolean_t fop_failed = _gf_false;
+ unsigned char *pre_op_sources = NULL;
+ int i = 0;
+
+ local = frame->local;
+ priv = this->private;
+ pre_op_sources = local->transaction.pre_op_sources;
+
+ if (priv->arbiter_count != 1 || local->op_ret < 0)
+ return;
+
+ /* If the fop failed on the brick, it is not a source. */
+ for (i = 0; i < priv->child_count; i++)
+ if (local->transaction.failed_subvols[i])
+ pre_op_sources[i] = 0;
+
+ switch (AFR_COUNT (pre_op_sources, priv->child_count)) {
+ case 1:
+ if (pre_op_sources[ARBITER_BRICK_INDEX])
+ fop_failed = _gf_true;
+ break;
+ case 0:
+ fop_failed = _gf_true;
+ break;
+ }
+
+ if (fop_failed) {
+ local->op_ret = -1;
+ local->op_errno = ENOTCONN;
+ }
+
+ return;
+}
+
+void
+afr_txn_arbitrate_fop (call_frame_t *frame, xlator_t *this)
+{
+ afr_local_t *local = NULL;
+ afr_private_t *priv = NULL;
+ int pre_op_sources_count = 0;
+
+ priv = this->private;
+ local = frame->local;
+
+ afr_compute_pre_op_sources (frame, this);
+ pre_op_sources_count = AFR_COUNT (local->transaction.pre_op_sources,
+ priv->child_count);
+
+ /* If arbiter is the only source, do not proceed. */
+ if (pre_op_sources_count < 2 &&
+ local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+ local->internal_lock.lock_cbk = local->transaction.done;
+ local->op_ret = -1;
+ local->op_errno = ENOTCONN;
+ afr_restore_lk_owner (frame);
+ afr_unlock (frame, this);
+ } else {
+ local->transaction.fop (frame, this);
+ }
+
+ return;
+}
int
afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
{
afr_local_t *local = NULL;
+ afr_private_t *priv = NULL;
fd_t *fd = NULL;
local = frame->local;
+ priv = this->private;
fd = local->fd;
/* Perform fops with the lk-owner from top xlator.
@@ -172,12 +288,15 @@ afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
*/
if (fd)
afr_delayed_changelog_wake_up (this, fd);
- local->transaction.fop (frame, this);
+ if (priv->arbiter_count == 1) {
+ afr_txn_arbitrate_fop (frame, this);
+ } else {
+ local->transaction.fop (frame, this);
+ }
return 0;
}
-
static int
__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
{
@@ -372,11 +491,16 @@ afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
{
afr_private_t *priv = NULL;
afr_local_t *local = NULL;
+ int pre_op_count = 0;
int i = 0;
local = frame->local;
priv = this->private;
+ pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+ if (pre_op_count < priv->child_count)
+ return _gf_false;
+
for (i = 0; i < priv->child_count; i++) {
if (local->transaction.failed_subvols[i])
return _gf_false;
@@ -591,9 +715,6 @@ afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
else
need_undirty = _gf_true;
- //If the fop fails on all the subvols then pending markers are placed
- //for every subvol on all subvolumes. Which is nothing but split-brain.
- //Avoid this by not doing post-op in case of failures.
if (local->op_ret < 0) {
afr_changelog_post_op_done (frame, this);
goto out;
@@ -846,12 +967,22 @@ afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
{
afr_local_t *local = NULL;
+ afr_private_t *priv = NULL;
int call_count = -1;
+ int child_index = -1;
local = frame->local;
+ priv = this->private;
+ child_index = (long) cookie;
if (op_ret == -1)
- afr_transaction_fop_failed (frame, this, (long) cookie);
+ afr_transaction_fop_failed (frame, this, child_index);
+
+ if (priv->arbiter_count == 1 && !op_ret) {
+ if (xattr)
+ local->transaction.pre_op_xdata[child_index] =
+ dict_ref (xattr);
+ }
call_count = afr_frame_return (frame);
@@ -964,7 +1095,6 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
afr_local_t *local = NULL;
afr_internal_lock_t *int_lock = NULL;
unsigned char *locked_nodes = NULL;
- unsigned char *pending_subvols = NULL;
int idx = -1;
gf_boolean_t pre_nop = _gf_true;
dict_t *xdata_req = NULL;
@@ -975,15 +1105,13 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
- pending_subvols = alloca0 (priv->child_count);
-
for (i = 0; i < priv->child_count; i++) {
if (locked_nodes[i]) {
local->transaction.pre_op[i] = 1;
call_count++;
} else {
- pending_subvols[i] = 1;
- }
+ local->transaction.failed_subvols[i] = 1;
+ }
}
/* This condition should not be met with present code, as
@@ -1009,28 +1137,21 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
goto err;
}
- pre_nop = _gf_true;
-
if (afr_changelog_pre_op_inherit (frame, this))
goto next;
- if (call_count < priv->child_count) {
- /* For subvols we are not performing operation on,
- mark them as pending up-front along with the FOP
- so that we can safely defer unmarking dirty until
- later.
- */
- for (i = 0; i < priv->child_count; i++) {
- if (pending_subvols[i])
- local->pending[i][idx] = hton32(1);
- }
- ret = afr_set_pending_dict (priv, xdata_req,
- local->pending);
- if (ret < 0) {
- op_errno = ENOMEM;
- goto err;
- }
- pre_nop = _gf_false;
+ if (call_count < priv->child_count)
+ pre_nop = _gf_false;
+
+ /* Set an all-zero pending changelog so that in the cbk, we can get the
+ * current on-disk values. In a replica 3 volume with arbiter enabled,
+ * these values are needed to arrive at a go/ no-go of the fop phase to
+ * avoid ending up in split-brain.*/
+
+ ret = afr_set_pending_dict (priv, xdata_req, local->pending);
+ if (ret < 0) {
+ op_errno = ENOMEM;
+ goto err;
}
if (call_count > 1 &&