From d06692d1deec425f74747e2c463e56f7eca981c8 Mon Sep 17 00:00:00 2001 From: Anuradha Date: Wed, 25 Feb 2015 15:09:28 +0530 Subject: cluster/afr : enable inspection & resolution of files in split-brain Part 2/2 patch to enable users analyze and resolve split-brain. This patch enables : 1) Users to inspect the files in data and metadata split-brain. 2) Resolve the split-brain. Both using a series of setfattr commands. Consider a volume "test" with 2 bricks. 1) To inspect a file f1: setfattr -n replica.split-brain-choice -v test-client-0 f1 After the execution of this command, if no read_subvol is found, reads will be served from test-client-0 (corresponding to brick-0). 2) To resolve split-brain : setfattr -n replica.split-brain-heal-finalize -v test-client-0 f1 Execution of this command will lead to the resolution of data and metadata split-brain with subvol mentioned in the command (test-client-0 here) as the source and the rest as sink. Change-Id: Ia20f3ee5abd3119e3d54fcc599f1e55ac65fd179 BUG: 1191396 Signed-off-by: Anuradha Reviewed-on: http://review.gluster.org/9743 Reviewed-by: Pranith Kumar Karampuri Tested-by: Gluster Build System --- tests/basic/afr/split-brain-resolution.t | 86 ++++++++++++++++++++++++++++++++ tests/volume.rc | 6 +++ 2 files changed, 92 insertions(+) create mode 100644 tests/basic/afr/split-brain-resolution.t (limited to 'tests') diff --git a/tests/basic/afr/split-brain-resolution.t b/tests/basic/afr/split-brain-resolution.t new file mode 100644 index 00000000000..feb527a71b3 --- /dev/null +++ b/tests/basic/afr/split-brain-resolution.t @@ -0,0 +1,86 @@ +#!/bin/bash +. $(dirname $0)/../../include.rc +. $(dirname $0)/../../volume.rc +cleanup; + +function get_split_brain_status { + local path=$1 + echo `getfattr -n replica.split-brain-status $path` | cut -f2 -d"=" | sed -e 's/^"//' -e 's/"$//' +} + +TEST glusterd +TEST pidof glusterd +TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{0,1} +TEST $CLI volume start $V0 + +#Disable self-heal-daemon +TEST $CLI volume set $V0 cluster.self-heal-daemon off + +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 --entry-timeout=0 $M0; + +TEST `echo "some-data" > $M0/data-split-brain.txt` +TEST `echo "some-data" > $M0/metadata-split-brain.txt` + +#Create data and metadata split-brain +TEST kill_brick $V0 $H0 $B0/${V0}0 + +TEST `echo "brick1_alive" > $M0/data-split-brain.txt` +TEST setfattr -n user.test -v brick1 $M0/metadata-split-brain.txt + +TEST $CLI volume start $V0 force +TEST kill_brick $V0 $H0 $B0/${V0}1 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 + +TEST `echo "brick0_alive" > $M0/data-split-brain.txt` +TEST setfattr -n user.test -v brick0 $M0/metadata-split-brain.txt + +TEST $CLI volume start $V0 force +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0 +EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1 + +EXPECT 4 afr_get_pending_heal_count $V0 + +TEST ! cat $M0/data-split-brain.txt +TEST ! getxattr -n user.test $M0/metadata-split-brain.txt + +#Inspect file in data-split-brain +EXPECT "data-split-brain:yes metadata-split-brain:no Choices:patchy-client-0,patchy-client-1" get_split_brain_status $M0/data-split-brain.txt +TEST setfattr -n replica.split-brain-choice -v $V0-client-0 $M0/data-split-brain.txt + +#Should now be able to read the contents of data-split-brain.txt +EXPECT "brick0_alive" cat $M0/data-split-brain.txt + +TEST setfattr -n replica.split-brain-choice -v $V0-client-1 $M0/data-split-brain.txt + +#Should now be able to read the contents of data-split-brain.txt +EXPECT "brick1_alive" cat $M0/data-split-brain.txt + +#Inspect the file in metadata-split-brain +EXPECT "data-split-brain:no metadata-split-brain:yes Choices:patchy-client-0,patchy-client-1" get_split_brain_status $M0/metadata-split-brain.txt +TEST setfattr -n replica.split-brain-choice -v $V0-client-0 $M0/metadata-split-brain.txt + +EXPECT "brick0" get_text_xattr user.test $M0/metadata-split-brain.txt + +TEST setfattr -n replica.split-brain-choice -v $V0-client-1 $M0/metadata-split-brain.txt +EXPECT "brick1" get_text_xattr user.test $M0/metadata-split-brain.txt + +#Check that setting split-brain-choice to "none" results in EIO again +TEST setfattr -n replica.split-brain-choice -v none $M0/metadata-split-brain.txt +TEST setfattr -n replica.split-brain-choice -v none $M0/data-split-brain.txt +TEST ! getxattr -n user.test $M0/metadata-split-brain.txt +TEST ! cat $M0/data-split-brain.txt + +#Negative test cases should fail +TEST ! setfattr -n replica.split-brain-choice -v $V0-client-4 $M0/data-split-brain.txt +TEST ! setfattr -n replica.split-brain-heal-finalize -v $V0-client-4 $M0/metadata-split-brain.txt + +#Heal the files +TEST setfattr -n replica.split-brain-heal-finalize -v $V0-client-0 $M0/metadata-split-brain.txt +TEST setfattr -n replica.split-brain-heal-finalize -v $V0-client-1 $M0/data-split-brain.txt + +EXPECT "brick0" get_text_xattr user.test $M0/metadata-split-brain.txt +EXPECT "brick1_alive" cat $M0/data-split-brain.txt + +EXPECT 0 afr_get_pending_heal_count $V0 + +cleanup; diff --git a/tests/volume.rc b/tests/volume.rc index 6abf68dc75c..1276dccdbae 100644 --- a/tests/volume.rc +++ b/tests/volume.rc @@ -242,6 +242,12 @@ function gf_gfid_xattr_to_str { echo "${xval:2:8}-${xval:10:4}-${xval:14:4}-${xval:18:4}-${xval:22:12}" } +function get_text_xattr { + local key=$1 + local path=$2 + getfattr -d -m. -e text $path 2>/dev/null | grep $key | cut -f2 -d'=' +} + function gf_check_file_opened_in_brick { vol=$1 host=$2 -- cgit