summaryrefslogtreecommitdiffstats
path: root/tests/basic/afr/split-brain-healing.t
blob: 8b7eaffb268d3070cf9898475ef3cd3a5367cdaa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
#!/bin/bash

#Test the split-brain resolution CLI commands.
. $(dirname $0)/../../include.rc
. $(dirname $0)/../../volume.rc

function get_replicate_subvol_number {
        local filename=$1
        #get_backend_paths
        if [ -f $B0/${V0}1/$filename ]
        then
                echo 0
        elif [ -f $B0/${V0}3/$filename ]
        then    echo 1
        else
                echo -1
        fi
}

cleanup;

AREQUAL_PATH=$(dirname $0)/../../utils
CFLAGS=""
test "`uname -s`" != "Linux" && {
    CFLAGS="$CFLAGS -I$(dirname $0)/../../../contrib/argp-standalone ";
    CFLAGS="$CFLAGS -L$(dirname $0)/../../../contrib/argp-standalone -largp ";
    CFLAGS="$CFLAGS -lintl";
}
build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST $CLI volume set $V0 cluster.data-self-heal off
TEST $CLI volume set $V0 cluster.metadata-self-heal off
TEST $CLI volume set $V0 cluster.entry-self-heal off
TEST $CLI volume start $V0
TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0

cd $M0
for i in {1..10}
do
        echo "Initial content">>file$i
done

replica_0_files_list=(`ls $B0/${V0}1|grep -v '^\.'`)
replica_1_files_list=(`ls $B0/${V0}3|grep -v '^\.'`)

############ Create data split-brain in the files. ###########################
TEST kill_brick $V0 $H0 $B0/${V0}1
for file in ${!replica_0_files_list[*]}
do
        echo "B1 is down">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}3
for file in ${!replica_1_files_list[*]}
do
        echo "B3 is down">>${replica_1_files_list[$file]}
done

SMALLER_FILE_SIZE=$(stat -c %s file1)

TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2

TEST kill_brick $V0 $H0 $B0/${V0}2
for file in ${!replica_0_files_list[*]}
do
        echo "B2 is down">>${replica_0_files_list[$file]}
        echo "appending more content to make it the bigger file">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}4
for file in ${!replica_1_files_list[*]}
do
        echo "B4 is down">>${replica_1_files_list[$file]}
        echo "appending more content to make it the bigger file">>${replica_1_files_list[$file]}
done

BIGGER_FILE_SIZE=$(stat -c %s file1)

TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3


############### Acessing the files should now give EIO. ###############################
TEST ! cat file1
TEST ! cat file2
TEST ! cat file3
TEST ! cat file4
TEST ! cat file5
TEST ! cat file6
TEST ! cat file7
TEST ! cat file8
TEST ! cat file9
TEST ! cat file10
###################
TEST $CLI volume set $V0 cluster.self-heal-daemon on
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 3

################ Heal file1 using the bigger-file option  ##############
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "0" echo $?
EXPECT $BIGGER_FILE_SIZE stat -c %s file1

################ Heal file2 using the bigger-file option and its gfid ##############
subvolume=$(get_replicate_subvol_number file2)
if [ $subvolume == 0 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}1/file2)
elif [ $subvolume == 1 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}3/file2)
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain bigger-file $GFIDSTR
EXPECT "0" echo $?

################ Heal file3 using the source-brick option  ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file3)
if [ $subvolume == 0 ]
then
        $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 /file3
elif [ $subvolume == 1 ]
then
        $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file3

################ Heal file4 using the source-brick option and it's gfid ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file4)
if [ $subvolume == 0 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}1/file4)
        GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
        $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 $GFIDSTR
elif [ $subvolume == 1 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}3/file4)
        GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
        $CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 $GFIDSTR
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file4

################ Heal file5 using the latest-mtime option  ##############
subvolume=$(get_replicate_subvol_number file5)
if [ $subvolume == 0 ]
then
        mtime1=$(stat -c %Y $B0/${V0}1/file5)
        mtime2=$(stat -c %Y $B0/${V0}2/file5)
        LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
        mtime1=$(stat -c %Y $B0/${V0}3/file5)
        mtime2=$(stat -c %Y $B0/${V0}4/file5)
        LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
$CLI volume heal $V0 split-brain latest-mtime /file5
EXPECT "0" echo $?

if [ $subvolume == 0 ]
then
        mtime1_after_heal=$(stat -c %Y $B0/${V0}1/file5)
        mtime2_after_heal=$(stat -c %Y $B0/${V0}2/file5)
elif [ $subvolume == 1 ]
then
        mtime1_after_heal=$(stat -c %Y $B0/${V0}3/file5)
        mtime2_after_heal=$(stat -c %Y $B0/${V0}4/file5)
fi

#TODO: To below comparisons on full sub-second resolution

TEST [ $LATEST_MTIME -eq $mtime1_after_heal ]
TEST [ $LATEST_MTIME -eq $mtime2_after_heal ]

mtime_mount_after_heal=$(stat -c %Y file5)

TEST [ $LATEST_MTIME -eq $mtime_mount_after_heal ]

################ Heal file6 using the latest-mtime option and its gfid  ##############
subvolume=$(get_replicate_subvol_number file6)
if [ $subvolume == 0 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}1/file6)
        mtime1=$(stat -c %Y $B0/${V0}1/file6)
        mtime2=$(stat -c %Y $B0/${V0}2/file6)
        LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
        GFID=$(gf_get_gfid_xattr $B0/${V0}3/file6)
        mtime1=$(stat -c %Y $B0/${V0}3/file6)
        mtime2=$(stat -c %Y $B0/${V0}4/file6)
        LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain latest-mtime $GFIDSTR
EXPECT "0" echo $?

if [ $subvolume == 0 ]
then
        mtime1_after_heal=$(stat -c %Y $B0/${V0}1/file6)
        mtime2_after_heal=$(stat -c %Y $B0/${V0}2/file6)
elif [ $subvolume == 1 ]
then
        mtime1_after_heal=$(stat -c %Y $B0/${V0}3/file6)
        mtime2_after_heal=$(stat -c %Y $B0/${V0}4/file6)
fi

#TODO: To below comparisons on full sub-second resolution

TEST [ $LATEST_MTIME -eq $mtime1_after_heal ]
TEST [ $LATEST_MTIME -eq $mtime2_after_heal ]

mtime_mount_after_heal=$(stat -c %Y file6)

TEST [ $LATEST_MTIME -eq $mtime_mount_after_heal ]

################ Heal remaining SB'ed files of replica_0 using B1 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1
EXPECT "0" echo $?

################ Heal remaining SB'ed files of replica_1 using B3 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}3
EXPECT "0" echo $?

############### Reading the files should now succeed. ###############################
TEST  cat file1
TEST  cat file2
TEST  cat file3
TEST  cat file4
TEST  cat file5
TEST  cat file6
TEST  cat file7
TEST  cat file8
TEST  cat file9
TEST  cat file10

################ File contents on the bricks must be same. ################################
TEST diff <(arequal-checksum -p $B0/$V01 -i .glusterfs) <(arequal-checksum -p $B0/$V02 -i .glusterfs)
TEST diff <(arequal-checksum -p $B0/$V03 -i .glusterfs) <(arequal-checksum -p $B0/$V04 -i .glusterfs)

############### Trying to heal files not in SB should fail. ###############################
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "1" echo $?
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
EXPECT "1" echo $?

cd -
TEST rm $AREQUAL_PATH/arequal-checksum
cleanup