summaryrefslogtreecommitdiffstats
path: root/xlators/cluster/afr
diff options
context:
space:
mode:
authorRavishankar N <ravishankar@redhat.com>2018-01-28 13:50:47 +0530
committerRavishankar N <ravishankar@redhat.com>2018-02-01 14:17:50 +0000
commit0e6e8216823c2d9dafb81aae0f6ee3497c23d140 (patch)
tree06481a13e40b18796cbef6a8248f539e6a739951 /xlators/cluster/afr
parentd663b9a323f34919da3f35bfc221a0aa91d9ab94 (diff)
afr: don't treat all cases all bricks being blamed as split-brain
Problem: We currently don't have a roll-back/undoing of post-ops if quorum is not met. Though the FOP is still unwound with failure, the xattrs remain on the disk. Due to these partial post-ops and partial heals (healing only when 2 bricks are up), we can end up in split-brain purely from the afr xattrs point of view i.e each brick is blamed by atleast one of the others. These scenarios are hit when there is frequent connect/disconnect of the client/shd to the bricks while I/O or heal are in progress. Fix: Instead of undoing the post-op, pick a source based on the xattr values. If 2 bricks blame one, the blamed one must be treated as sink. If there is no majority, all are sources. Once we pick a source, self-heal will then do the heal instead of erroring out due to split-brain. Change-Id: I3d0224b883eb0945785ade0e9697a1c828aec0ae BUG: 1539358 Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Diffstat (limited to 'xlators/cluster/afr')
-rw-r--r--xlators/cluster/afr/src/afr-self-heal-common.c51
-rw-r--r--xlators/cluster/afr/src/afr-self-heal-data.c6
2 files changed, 48 insertions, 9 deletions
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 311da68e8ee..7e2a6027126 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1455,6 +1455,36 @@ afr_does_witness_exist (xlator_t *this, uint64_t *witness)
return _gf_false;
}
+unsigned int
+afr_get_quorum_count (afr_private_t *priv)
+{
+ if (priv->quorum_count == AFR_QUORUM_AUTO) {
+ return priv->child_count/2 + 1;
+ } else {
+ return priv->quorum_count;
+ }
+}
+
+void
+afr_selfheal_post_op_failure_accounting (afr_private_t *priv, char *accused,
+ unsigned char *sources,
+ unsigned char *locked_on)
+{
+ int i = 0;
+ unsigned int quorum_count = 0;
+
+ if (AFR_COUNT (sources, priv->child_count) != 0)
+ return;
+
+ quorum_count = afr_get_quorum_count (priv);
+ for (i = 0; i < priv->child_count; i++) {
+ if ((accused[i] < quorum_count) && locked_on[i]) {
+ sources[i] = 1;
+ }
+ }
+ return;
+}
+
/*
* This function determines if a self-heal is required for a given inode,
* and if needed, in what direction.
@@ -1490,6 +1520,7 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
char *accused = NULL;/* Accused others without any self-accusal */
char *pending = NULL;/* Have pending operations on others */
char *self_accused = NULL; /* Accused itself */
+ int min_participants = -1;
priv = this->private;
@@ -1513,8 +1544,13 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
}
}
+ if (type == AFR_DATA_TRANSACTION) {
+ min_participants = priv->child_count;
+ } else {
+ min_participants = AFR_SH_MIN_PARTICIPANTS;
+ }
if (afr_success_count (replies,
- priv->child_count) < AFR_SH_MIN_PARTICIPANTS) {
+ priv->child_count) < min_participants) {
/* Treat this just like locks not being acquired */
return -ENOTCONN;
}
@@ -1530,11 +1566,10 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
for (i = 0; i < priv->child_count; i++) {
for (j = 0; j < priv->child_count; j++) {
if (matrix[i][j]) {
- if (!self_accused[i])
- accused[j] = 1;
-
- if (i != j)
- pending[i] = 1;
+ if (!self_accused[i])
+ accused[j] += 1;
+ if (i != j)
+ pending[i] += 1;
}
}
}
@@ -1575,6 +1610,10 @@ afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
}
}
+ if (type == AFR_DATA_TRANSACTION)
+ afr_selfheal_post_op_failure_accounting (priv, accused,
+ sources, locked_on);
+
/* If no sources, all locked nodes are sinks - split brain */
if (AFR_COUNT (sources, priv->child_count) == 0) {
for (i = 0; i < priv->child_count; i++) {
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 7880a611592..c033a8afc07 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -684,7 +684,7 @@ __afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
data_lock);
{
- if (ret < AFR_SH_MIN_PARTICIPANTS) {
+ if (ret < priv->child_count) {
gf_msg_debug (this->name, 0, "%s: Skipping "
"self-heal as only %d number "
"of subvolumes "
@@ -749,7 +749,7 @@ restore_time:
if (!is_arbiter_the_only_sink) {
ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
0, 0, data_lock);
- if (ret < AFR_SH_MIN_PARTICIPANTS) {
+ if (ret < priv->child_count) {
ret = -ENOTCONN;
did_sh = _gf_false;
goto skip_undo_pending;
@@ -878,7 +878,7 @@ afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
priv->sh_domain, 0, 0,
locked_on);
{
- if (ret < AFR_SH_MIN_PARTICIPANTS) {
+ if (ret < priv->child_count) {
gf_msg_debug (this->name, 0, "%s: Skipping "
"self-heal as only %d number of "
"subvolumes could be locked",