summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKrutika Dhananjay <kdhananj@redhat.com>2012-12-27 15:57:13 +0530
committerAnand Avati <avati@redhat.com>2013-02-08 19:13:36 -0800
commit7c0d35b2e7e234084d0414b3cf1e09969a43a677 (patch)
tree013d853e2139ed43875123ddec3eef4bc97e6d14
parent19de18219b93097ede8d14c218011a873ebd50ed (diff)
glusterd: harden 'volume start' staging to check for brick dirs' presence
PROBLEM: When the brick directory of a volume is absent on any of the servers, AND an attempt is made to start the volume, commit fails ONLY on the node where the brick dir is absent, leading to a split-brain like situation. FIX: Harden 'volume start' to check for the presence of brick directories at the time of staging, thereby preventing commit failure. Change-Id: I67faeb9afbd3aa76f08645924462db126bf7a977 BUG: 889996 Signed-off-by: Krutika Dhananjay <kdhananj@redhat.com> Reviewed-on: http://review.gluster.org/4365 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Anand Avati <avati@redhat.com>
-rw-r--r--libglusterfs/src/common-utils.c30
-rw-r--r--libglusterfs/src/common-utils.h3
-rw-r--r--tests/bugs/bug-889996.t19
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-ops.c62
4 files changed, 101 insertions, 13 deletions
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 75b97722513..19afeee3d1f 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -44,6 +44,7 @@
#include "stack.h"
#include "globals.h"
#include "lkowner.h"
+#include "syscall.h"
#ifndef AI_ADDRCONFIG
#define AI_ADDRCONFIG 0
@@ -113,6 +114,35 @@ out:
}
int
+gf_lstat_dir (const char *path, struct stat *stbuf_in)
+{
+ int ret = -1;
+ struct stat stbuf = {0,};
+
+ if (path == NULL) {
+ errno = EINVAL;
+ goto out;
+ }
+
+ ret = sys_lstat (path, &stbuf);
+ if (ret)
+ goto out;
+
+ if (!S_ISDIR (stbuf.st_mode)) {
+ errno = ENOTDIR;
+ ret = -1;
+ goto out;
+ }
+ ret = 0;
+
+out:
+ if (!ret && stbuf_in)
+ *stbuf_in = stbuf;
+
+ return ret;
+}
+
+int
log_base2 (unsigned long x)
{
int val = 0;
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index bbafd1fb039..d04df579b53 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -483,6 +483,9 @@ mkdir_p (char *path, mode_t mode, gf_boolean_t allow_symlinks);
* nr
*/
+int
+gf_lstat_dir (const char *path, struct stat *stbuf_in);
+
int32_t gf_roundup_power_of_two (int32_t nr);
/*
diff --git a/tests/bugs/bug-889996.t b/tests/bugs/bug-889996.t
new file mode 100644
index 00000000000..6b07d8918d0
--- /dev/null
+++ b/tests/bugs/bug-889996.t
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+cleanup;
+
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2};
+
+rm -rf $B0/${V0}1;
+
+TEST ! $CLI volume start $V0;
+EXPECT 0 online_brick_count;
+
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index cf424abb2d5..fb1832f0d47 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -814,6 +814,9 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
char msg[2048];
glusterd_conf_t *priv = NULL;
xlator_t *this = NULL;
+ uuid_t volume_id = {0,};
+ char volid[50] = {0,};
+ char xattr_volid[50] = {0,};
this = THIS;
GF_ASSERT (this);
@@ -828,8 +831,6 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
if (!exists) {
snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
- gf_log (this->name, GF_LOG_ERROR, "%s", msg);
- *op_errstr = gf_strdup (msg);
ret = -1;
goto out;
}
@@ -845,6 +846,15 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
if (ret)
goto out;
+ if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+ if (glusterd_is_volume_started (volinfo)) {
+ snprintf (msg, sizeof (msg), "Volume %s already "
+ "started", volname);
+ ret = -1;
+ goto out;
+ }
+ }
+
list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
ret = glusterd_resolve_brick (brickinfo);
if (ret) {
@@ -853,22 +863,48 @@ glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr)
goto out;
}
- if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
- if (glusterd_is_volume_started (volinfo)) {
- snprintf (msg, sizeof (msg), "Volume %s already"
- " started", volname);
- gf_log (this->name, GF_LOG_ERROR, "%s", msg);
- *op_errstr = gf_strdup (msg);
- ret = -1;
- goto out;
- }
+ if (uuid_compare (brickinfo->uuid, MY_UUID))
+ continue;
+
+ if (volinfo->backend == GD_VOL_BK_BD)
+ continue;
+
+ ret = gf_lstat_dir (brickinfo->path, NULL);
+ if (ret) {
+ snprintf (msg, sizeof (msg), "Failed to find "
+ "brick directory %s for volume %s. "
+ "Reason : %s", brickinfo->path,
+ volname, strerror (errno));
+ goto out;
+ }
+ ret = sys_lgetxattr (brickinfo->path, GF_XATTR_VOL_ID_KEY,
+ volume_id, 16);
+ if (ret < 0) {
+ snprintf (msg, sizeof (msg), "Failed to get "
+ "extended attribute %s for brick dir %s. "
+ "Reason : %s", GF_XATTR_VOL_ID_KEY,
+ brickinfo->path, strerror (errno));
+ ret = -1;
+ goto out;
+ }
+ if (uuid_compare (volinfo->volume_id, volume_id)) {
+ snprintf (msg, sizeof (msg), "Volume id mismatch for "
+ "brick %s:%s. Expected volume id %s, "
+ "volume id %s found", brickinfo->hostname,
+ brickinfo->path,
+ uuid_utoa_r (volinfo->volume_id, volid),
+ uuid_utoa_r (volume_id, xattr_volid));
+ ret = -1;
+ goto out;
}
}
ret = 0;
out:
- gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
+ if (ret && (msg[0] != '\0')) {
+ gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+ *op_errstr = gf_strdup (msg);
+ }
return ret;
}