#!/bin/bash #Self-heal tests . $(dirname $0)/../../include.rc . $(dirname $0)/../../volume.rc cleanup; function create_file_with_gfids { local b1_path=$1 local b2_path=$2 local filename=$3 local gfid=$(get_random_gfid) touch $b1_path/$filename setfattr -n trusted.gfid -v $gfid $b1_path/$filename touch $b2_path/$filename setfattr -n trusted.gfid -v $gfid $b2_path/$filename } function is_heal_done { local f1_path="${1}/${3}" local f2_path="${2}/${3}" local zero_xattr="000000000000000000000000" local size1=$(stat -c "%s" $f1_path) local size2=$(stat -c "%s" $f2_path) local diff=$((size1-size2)) local xattr11=$(get_hex_xattr trusted.afr.$V0-client-0 $f1_path) local xattr12=$(get_hex_xattr trusted.afr.$V0-client-1 $f1_path) local xattr21=$(get_hex_xattr trusted.afr.$V0-client-0 $f2_path) local xattr22=$(get_hex_xattr trusted.afr.$V0-client-1 $f2_path) local dirty1=$(get_hex_xattr trusted.afr.dirty $f1_path) local dirty2=$(get_hex_xattr trusted.afr.dirty $f2_path) if [ -z $xattr11 ]; then xattr11="000000000000000000000000"; fi if [ -z $xattr12 ]; then xattr12="000000000000000000000000"; fi if [ -z $xattr21 ]; then xattr21="000000000000000000000000"; fi if [ -z $xattr22 ]; then xattr22="000000000000000000000000"; fi if [ -z $dirty1 ]; then dirty1="000000000000000000000000"; fi if [ -z $dirty2 ]; then dirty2="000000000000000000000000"; fi if [ "${diff}${xattr11}${xattr12}${xattr21}${xattr22}${dirty1}${dirty2}" == "0${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}${zero_xattr}" ]; then echo "Y" else echo "N" fi } function print_pending_heals { local result=":" for i in "$@"; do if [ "N" == $(is_heal_done $B0/brick0 $B0/brick1 $i) ]; then result="$result:$i" fi done #To prevent any match for EXPECT_WITHIN, print a char non-existent in file-names if [ $result == ":" ]; then result="~"; fi echo $result } TEST glusterd TEST pidof glusterd TEST $CLI volume create $V0 replica 2 $H0:$B0/brick{0,1} TEST $CLI volume set $V0 cluster.background-self-heal-count 0 TEST create_file_with_gfids $B0/brick0 $B0/brick1 pending-changelog TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000010000000000000000 $B0/brick0/pending-changelog TEST "echo abc > $B0/brick1/pending-changelog" TEST create_file_with_gfids $B0/brick0 $B0/brick1 biggest-file-source.txt TEST "echo abc > $B0/brick0/biggest-file-source.txt" TEST "echo abcd > $B0/brick1/biggest-file-source.txt" TEST create_file_with_gfids $B0/brick0 $B0/brick1 biggest-file-more-prio-than-changelog.txt TEST "echo abc > $B0/brick0/biggest-file-more-prio-than-changelog.txt" TEST "echo abcd > $B0/brick1/biggest-file-more-prio-than-changelog.txt" TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick0/biggest-file-more-prio-than-changelog.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 same-size-more-prio-to-changelog.txt TEST "echo abc > $B0/brick0/same-size-more-prio-to-changelog.txt" TEST "echo def > $B0/brick1/same-size-more-prio-to-changelog.txt" TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick0/same-size-more-prio-to-changelog.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 size-and-witness-same.txt TEST "echo abc > $B0/brick0/size-and-witness-same.txt" TEST "echo def > $B0/brick1/size-and-witness-same.txt" TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick0/size-and-witness-same.txt TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick1/size-and-witness-same.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 split-brain.txt TEST "echo abc > $B0/brick0/split-brain.txt" TEST "echo def > $B0/brick1/split-brain.txt" TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick0/split-brain.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/split-brain.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 split-brain-all-dirty.txt TEST "echo abc > $B0/brick0/split-brain-all-dirty.txt" TEST "echo def > $B0/brick1/split-brain-all-dirty.txt" TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick0/split-brain-all-dirty.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/split-brain-all-dirty.txt TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick0/split-brain-all-dirty.txt TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick1/split-brain-all-dirty.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 split-brain-with-dirty.txt TEST "echo abc > $B0/brick0/split-brain-with-dirty.txt" TEST "echo def > $B0/brick1/split-brain-with-dirty.txt" TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick0/split-brain-with-dirty.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/split-brain-with-dirty.txt TEST setfattr -n trusted.afr.dirty -v 0x000000200000000000000000 $B0/brick1/split-brain-with-dirty.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 self-accusing-vs-source.txt TEST "echo def > $B0/brick1/self-accusing-vs-source.txt" TEST "echo abc > $B0/brick0/self-accusing-vs-source.txt" TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/self-accusing-vs-source.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick1/self-accusing-vs-source.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick0/self-accusing-vs-source.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 self-accusing-both.txt TEST "echo abc > $B0/brick0/self-accusing-both.txt" TEST "echo def > $B0/brick1/self-accusing-both.txt" TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick0/self-accusing-both.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick0/self-accusing-both.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/self-accusing-both.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick1/self-accusing-both.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 self-accusing-vs-innocent.txt TEST "echo abc > $B0/brick0/self-accusing-vs-innocent.txt" TEST "echo def > $B0/brick1/self-accusing-vs-innocent.txt" TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick1/self-accusing-vs-innocent.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 self-accusing-bigger-exists.txt TEST "echo abc > $B0/brick0/self-accusing-bigger-exists.txt" TEST "echo def > $B0/brick1/self-accusing-bigger-exists.txt" TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick0/self-accusing-bigger-exists.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000300000000000000000 $B0/brick0/self-accusing-bigger-exists.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/self-accusing-bigger-exists.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick1/self-accusing-bigger-exists.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 size-more-prio-than-self-accused.txt TEST "echo abc > $B0/brick0/size-more-prio-than-self-accused.txt" TEST "echo defg > $B0/brick1/size-more-prio-than-self-accused.txt" TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick0/size-more-prio-than-self-accused.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000300000000000000000 $B0/brick0/size-more-prio-than-self-accused.txt TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick1/size-more-prio-than-self-accused.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000200000000000000000 $B0/brick1/size-more-prio-than-self-accused.txt TEST create_file_with_gfids $B0/brick0 $B0/brick1 v1-dirty.txt TEST "echo abc > $B0/brick0/v1-dirty.txt" TEST "echo def > $B0/brick1/v1-dirty.txt" TEST setfattr -n trusted.afr.$V0-client-0 -v 0x000000200000000000000000 $B0/brick0/v1-dirty.txt TEST setfattr -n trusted.afr.$V0-client-1 -v 0x000000100000000000000000 $B0/brick1/v1-dirty.txt TEST $CLI volume start $V0 TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 --entry-timeout=0 --attribute-timeout=0; cd $M0 TEST stat pending-changelog biggest-file-source.txt biggest-file-more-prio-than-changelog.txt same-size-more-prio-to-changelog.txt size-and-witness-same.txt self-accusing-vs-source.txt self-accusing-both.txt self-accusing-vs-innocent.txt self-accusing-bigger-exists.txt size-more-prio-than-self-accused.txt v1-dirty.txt cd - EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1 TEST $CLI volume heal $V0 full EXPECT_WITHIN $HEAL_TIMEOUT "~" print_pending_heals pending-changelog biggest-file-source.txt biggest-file-more-prio-than-changelog.txt same-size-more-prio-to-changelog.txt size-and-witness-same.txt self-accusing-vs-source.txt self-accusing-both.txt self-accusing-vs-innocent.txt self-accusing-bigger-exists.txt size-more-prio-than-self-accused.txt v1-dirty.txt EXPECT "N" is_heal_done $B0/brick0 $B0/brick1 split-brain.txt EXPECT "N" is_heal_done $B0/brick0 $B0/brick1 split-brain-all-dirty.txt EXPECT "N" is_heal_done $B0/brick0 $B0/brick1 split-brain-with-dirty.txt EXPECT "0" stat -c "%s" $M0/pending-changelog TEST cmp $B0/brick0/pending-changelog $B0/brick1/pending-changelog EXPECT "abcd" cat $M0/biggest-file-source.txt TEST cmp $B0/brick0/biggest-file-source.txt $B0/brick1/biggest-file-source.txt EXPECT "abcd" cat $M0/biggest-file-more-prio-than-changelog.txt TEST cmp $B0/brick0/biggest-file-more-prio-than-changelog.txt $B0/brick1/biggest-file-more-prio-than-changelog.txt EXPECT "abc" cat $M0/same-size-more-prio-to-changelog.txt TEST cmp $B0/brick0/same-size-more-prio-to-changelog.txt $B0/brick1/same-size-more-prio-to-changelog.txt EXPECT "(abc|def)" cat $M0/size-and-witness-same.txt TEST cmp $B0/brick0/size-and-witness-same.txt $B0/brick1/size-and-witness-same.txt TEST ! cat $M0/split-brain.txt TEST ! cat $M0/split-brain-all-dirty.txt TEST ! cat $M0/split-brain-with-dirty.txt EXPECT "abc" cat $M0/self-accusing-vs-source.txt TEST cmp $B0/brick0/self-accusing-vs-source.txt $B0/brick1/self-accusing-vs-source.txt EXPECT "(abc|def)" cat $M0/self-accusing-both.txt TEST cmp $B0/brick0/self-accusing-both.txt $B0/brick1/self-accusing-both.txt EXPECT "def" cat $M0/self-accusing-vs-innocent.txt TEST cmp $B0/brick0/self-accusing-vs-innocent.txt $B0/brick1/self-accusing-vs-innocent.txt EXPECT "abc" cat $M0/self-accusing-bigger-exists.txt TEST cmp $B0/brick0/self-accusing-bigger-exists.txt $B0/brick1/self-accusing-bigger-exists.txt EXPECT "defg" cat $M0/size-more-prio-than-self-accused.txt TEST cmp $B0/brick0/size-more-prio-than-self-accused.txt $B0/brick1/size-more-prio-than-self-accused.txt EXPECT "abc" cat $M0/v1-dirty.txt TEST cmp $B0/brick0/v1-dirty.txt $B0/brick1/v1-dirty.txt cleanup;