blob: 945893d0f26263e7d54a2f31ed433c824d934522 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
|
#!/usr/bin/env bash
#************************************************************************#
# #
# Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com> #
# This file is part of gluster-block. #
# #
# This file is licensed to you under your choice of the GNU Lesser #
# General Public License, version 3 or any later version (LGPLv3 or #
# later), or the GNU General Public License, version 2 (GPLv2), in all #
# cases as published by the Free Software Foundation. #
# #
#************************************************************************#
# How to run this script: #
# ---------------------- #
# $ ./replace-node.sh ${MOUNTPOINT} ${OLDNODE} ${NEWNODE} #
# #
#************************************************************************#
LOGDIR="/var/log/gluster-block/"
function schedule_terminate () {
echo -e "\nholdon! we have received signal, this program will be terminated in the next safe point ..\n"
printLog "Terminating script in the signal handler..."
SAFE_EXIT=1
fdClose ${fd}
}
function getAvailableFd() {
for i in {1..65536}; do
if [ ! -e /proc/$$/fd/${i} ]; then
echo ${i};
return 0;
fi
done
return 1;
}
function fdOpen() {
local fd=${1}
local path=${2}
# open in read mode
eval "exec ${fd}<${path}"
}
function fdClose() {
local fd=${1}
eval "exec ${fd}>&-"
}
function printLog() {
local msg=${1}
echo "[$(date -u +'%Y-%m-%d %I:%M:%S')] ${msg}" >> ${LOGDIR}/gluster-block-replace.log
}
function readMetaInfo() {
while read line; do
METADATA+=(["$(echo ${line} | cut -d: -f1)"]=$(echo ${line} | cut -d' ' -f2 | tr -d " \n"))
done < ${1}
}
function isStrPartOfArray() {
local args=("${@}")
local str=${args[0]}
local array=${args[@]:1}
for i in ${array[@]}; do
if [ "${str}" == "${i}" ]; then
return 0;
fi
done
return 1;
}
function isStatusValid() {
local status=${1}
case "${status}" in
"CONFIGSUCCESS" | "CLEANUPINPROGRESS" | "AUTHENFORCEING" | "AUTHENFORCED" | "AUTHENFORCEFAIL" | "AUTHCLEARENFORCED" | "AUTHCLEARENFORCEING" | "AUTHCLEARENFORCEFAIL")
return 0;
;;
esac
return 1;
}
function getValidHosts() {
eval "declare -A metadata="${1#*=}
local validHostsArr=()
local commonKeys=(VOLUME GBID SIZE HA ENTRYCREATE PASSWORD)
for index in ${!metadata[@]}; do
isStrPartOfArray "${index}" "${commonKeys[@]}"
ret=$?
if [[ ${ret} -ne 0 ]]; then
isStatusValid ${metadata["${index}"]}
ret=$?
if [[ ${ret} -ne 1 ]]; then
validHostsArr+=("${index}")
fi
fi
done
unset metadata
echo "${validHostsArr[@]}" # return
}
function runCreate() {
eval "declare -A metadata="${1#*=}
local block_name=${2}
local old_node=${3}
local new_node=${4}
local iqn_prefix="iqn.2016-12.org.gluster-block"
local gbid=${metadata["GBID"]}
local hosts=( $(getValidHosts "$(declare -p metadata)") )
local subcmd1="\$(targetcli /backstores/user:glfs ls | grep ' ${block_name} ' -c)"
local subcmd2="targetcli set global auto_add_default_portal=false auto_enable_tpgt=false loglevel_file=info logfile=${LOGFILE}\n
targetcli /backstores/user:glfs create ${block_name} ${metadata["SIZE"]} ${metadata["VOLUME"]}@${new_node}/block-store/${gbid} ${gbid}\n
targetcli /backstores/user:glfs/${block_name} set attribute cmd_time_out=0\n
targetcli /iscsi create ${iqn_prefix}:${gbid}\n"
for index in ${!hosts[@]}; do
NODE=${hosts[$index]}
if [[ "${NODE}" == "${old_node}" ]]; then
NODE=${new_node}
fi
no=$(expr ${index} + 1)
if [[ ${no} -ne 1 ]]; then
subcmd2+="targetcli /iscsi/${iqn_prefix}:${metadata["GBID"]} create tpg${no}\n"
fi
if [[ "${NODE}" == "${new_node}" ]]; then
subcmd2+="targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no}/luns create /backstores/user:glfs/${block_name}\n
targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no}/portals create ${NODE}\n
targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no} enable\n
targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no} set attribute generate_node_acls=1 demo_mode_write_protect=0\n"
else
subcmd2+="targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no}/luns create /backstores/user:glfs/${block_name}\n
targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no}/portals create ${NODE}\n
targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no} set attribute tpg_enabled_sendtargets=0 generate_node_acls=1 demo_mode_write_protect=0\n"
fi
if [[ "x${metadata["PASSWORD"]}" != "x" ]]; then
subcmd2+="targetcli /iscsi/${iqn_prefix}:${gbid}/tpg${no} set auth userid=${gbid} password=${metadata["PASSWORD"]}\n"
fi
done
subcmd2+="targetcli / saveconfig"
cmd="if [[ ${subcmd1} -eq 0 ]]; then ${subcmd2}; else echo "skipped"; fi"
unset metadata
IFS=' '
eval $(printf "${cmd}") # exec
}
function runReplace() {
eval "declare -A metadata="${1#*=}
local old_node=${2}
local new_node=${3}
local iqn_prefix="iqn.2016-12.org.gluster-block"
local gbid=${metadata["GBID"]}
local tpgno=$(targetcli /iscsi/${iqn_prefix}:${gbid} ls | grep -e tpg -e " ${old_node}:" | grep -B1 " ${old_node}:" | grep -o "tpg[0-9]*")
local subcmd="targetcli /iscsi/${iqn_prefix}:${gbid}/${tpgno}/portals create ${new_node};
targetcli /iscsi/${iqn_prefix}:${gbid}/${tpgno}/portals delete ip_address=${old_node} ip_port=3260;
targetcli / saveconfig"
cmd="if [[ "x" != "${tpgno}x" ]]; then ${subcmd}; else echo "skipped"; fi"
unset metadata
IFS=' '
eval $(printf "${cmd}") # exec
}
function runDelete() {
eval "declare -A metadata="${1#*=}
local block_name=${2}
local iqn_prefix="iqn.2016-12.org.gluster-block"
local gbid=${metadata["GBID"]}
local subcmd1="\$(targetcli /backstores/user:glfs ls | grep ' ${block_name} ' | grep '/${gbid} ' -c)"
local subcmd2="targetcli /backstores/user:glfs delete ${block_name};
targetcli /iscsi delete ${iqn_prefix}:${gbid};
targetcli / saveconfig"
cmd="if [[ ${subcmd1} -gt 0 ]]; then ${subcmd2}; else echo "skipped"; fi"
unset metadata
IFS=' '
eval $(printf "${cmd}") # exec
}
function parseCreateOutput() {
eval "declare -A metadata="${1#*=}
local block_name=${2}
local new_node=${3}
local output=()
output="$(cat /tmp/gb_create)"
if [[ "${output}" == "skipped" ]]; then
printLog "WARNING: create: creating ${block_name} on ${new_node} skipped"
return 1;
fi
output=("$(cat /tmp/gb_create | awk '/Created user-backed storage/{flag=1} /Created TPG 1/{flag=0} flag')")
local i=1;
while [ $i -le ${metadata["HA"]} ]; do
output+=("$(cat /tmp/gb_create | awk "/Created TPG $i/{flag=1} /Created TPG $(expr $i + 1)/{flag=0} flag")")
i=`expr $i + 1`
done
for index in ${!output[@]}; do
if [[ ${index} -eq 0 ]]; then
if [[ ${output["${index}"]} != *"Created user-backed storage object ${block_name} size ${metadata["SIZE"]}."* ]]; then
printLog "ERROR: create: create user-backed storage object for ${block_name} with size ${metadata["SIZE"]} failed"
return 1;
fi
if [[ ${output["${index}"]} != *"Parameter cmd_time_out is now '0'."* ]]; then
printLog "ERROR: create: setting cmd_time_out=0 for ${block_name} failed"
return 1;
fi
if [[ ${output["${index}"]} != *"Created target iqn.2016-12.org.gluster-block:${metadata["GBID"]}."* ]]; then
printLog "ERROR: create: create target for ${block_name} failed"
return 1;
fi
else
if [[ ${output["${index}"]} != *"Created TPG ${index}."* ]]; then
printLog "ERROR: create: create TPG ${index} failed for ${block_name}"
return 1;
fi
if [[ ${output["${index}"]} != *"Created LUN 0."* ]]; then
printLog "ERROR: create: creation of LUN 0 for ${block_name} on tpg ${index} failed"
return 1;
fi
if [[ ${output["${index}"]} != *"Using default IP port 3260"* ]]; then
printLog "ERROR: create: using default portal failed for ${block_name} on tpg ${index}"
return 1;
fi
if [[ ${output["${index}"]} != *"Created network portal"* ]]; then
printLog "ERROR: create: creation of portal failed for ${block_name} on tpg ${index}"
return 1;
fi
if [[ ${output["${index}"]} != *"Parameter generate_node_acls is now '1'."* ]]; then
printLog "ERROR: create: setting generate_node_acls=1 failed for ${block_name} on tpg ${index}"
return 1;
fi
if [[ ${output["${index}"]} != *"Parameter demo_mode_write_protect is now '0'."* ]]; then
printLog "ERROR: create: setting demo_mode_write_protect=0 failed for ${block_name} on tpg ${index}"
return 1;
fi
if [[ ${output["${index}"]} == *"${new_node}"* ]]; then
if [[ ${output["${index}"]} != *"The TPGT has been enabled."* ]]; then
printLog "ERROR: create: enabling TPGT failed for ${block_name} on tpg ${index} & portal ${new_node}"
return 1;
fi
else
if [[ ${output["${index}"]} != *"Parameter tpg_enabled_sendtargets is now '0'."* ]]; then
printLog "ERROR: create: setting tpg_enabled_sendtargets=0 failed for ${block_name} on tpg ${index}"
return 1;
fi
fi
fi
done
unset metadata
return 0;
}
function parseReplaceOutput() {
local output=${1}
local block_name=${2}
local old_node=${3}
local new_node=${4}
if [[ "${output}" == "skipped" ]]; then
printLog "WARNING: replace: replacing network portal ${old_node}:3260 for ${block_name} skipped"
return 1;
fi
if [[ ${output} != *"Deleted network portal ${old_node}:3260"* ]]; then
printLog "ERROR: replace: deleting network portal ${old_node}:3260 for ${block_name} failed"
return -1;
fi
if [[ ${output} != *"Using default IP port 3260"* ]]; then
printLog "ERROR: replace: using default portal for ${block_name} failed"
return -1;
fi
if [[ ${output} != *"Created network portal ${new_node}:3260."* ]]; then
printLog "ERROR: replace: creation of portal ${new_node} for ${block_name} failed"
return -1;
fi
return 0;
}
function parseDeleteOutput() {
local output=${1}
local block_name=${2}
local gbid=${3}
local iqn_prefix="iqn.2016-12.org.gluster-block"
if [[ "${output}" == "skipped" ]]; then
printLog "WARNING: delete: deleting storage object ${block_name} skipped"
return 1;
fi
if [[ ${output} != *"Deleted storage object ${block_name}."* ]]; then
printLog "ERROR: delete: deleting storage object ${block_name} failed"
return -1;
fi
if [[ ${output} != *"Deleted Target ${iqn_prefix}:${gbid}."* ]]; then
printLog "ERROR: delete: deleting target ${iqn_prefix}:${gbid} of ${block_name} failed"
return -1;
fi
return 0;
}
function readMetaInfo() {
while read line; do
METADATA+=(["$(echo ${line} | cut -d: -f1)"]=$(echo ${line} | cut -d' ' -f2 | tr -d " \n"))
done < ${1}
}
function takeLockInit() {
local fd=${1}
# ignore signals till trap setting
trap '' INT TERM QUIT
flock -x ${fd}
if [[ ${?} -ne 0 ]]; then
printLog "failed to take lock"
exit 1;
fi
}
function releaseLockExit() {
local fd=${1}
flock -u ${fd}
if [[ ${?} -ne 0 ]]; then
printLog "failed to release lock"
exit 1;
fi
# trap open for very short duration
trap schedule_terminate INT TERM QUIT
sleep 1;
}
function usage() {
cat <<USAGE
Usage:
./replace-node.sh \${MOUNTPOINT} \${OLDNODE} \${NEWNODE}
USAGE
}
# start main
if [[ ${#} -ne 3 ]]; then
usage
exit 1;
fi
MOUNTPOINT=${1}
OLDNODE=${2}
NEWNODE=${3}
DIR=${MOUNTPOINT}/block-meta/
if [[ ! -d "${DIR}" ]]; then
echo "Fuse mount point, does not have directory '${DIR}'"
usage
exit 1;
fi
mkdir -p ${LOGDIR}
printLog "Starting replace node job, mount-point=${MOUNTPOINT} old-node=${OLDNODE} new-node=${NEWNODE}"
# Fetch an available fd
fd=$(getAvailableFd)
if [[ ${?} -ne 0 ]]; then
printLog "no free fd's available on the node"
exit 1;
fi
fdOpen ${fd} ${DIR}/meta.lock
SAFE_EXIT=0
for BLOCKNAME in $(ls ${DIR}); do
declare -A METADATA
# handle last signal (aka safe point)
if [[ ${SAFE_EXIT} -eq 1 ]]; then
echo "exiting in a safe point :)"
printLog "exiting in a safe point..."
exit 1;
fi
takeLockInit ${fd}
if [[ ${BLOCKNAME} == "meta.lock" ]]; then
continue;
fi
readMetaInfo "${DIR}/${BLOCKNAME}"
HOSTS=( $(getValidHosts "$(declare -p METADATA)") )
echo "blockname: ${BLOCKNAME}"
skip=0;
if [[ $(echo "${!METADATA[@]} " | grep -c " ${OLDNODE} ") -eq 0 ]]; then
echo "${OLDNODE} not configured for this block."
echo "-------------------------------------"
printLog "INFO: ${BLOCKNAME} not configured on ${OLDNODE}, skipping replace..."
skip=1;
fi
if [[ skip -eq 1 ]]; then
# unset variables
unset HOSTS
unset METADATA
releaseLockExit ${fd}
continue;
fi
ssh root@${NEWNODE} "LOGFILE=${LOGDIR}/gluster-block-configshell.log; $(declare -f isStrPartOfArray isStatusValid getValidHosts runCreate); runCreate '$(declare -p METADATA)' ${BLOCKNAME} ${OLDNODE} ${NEWNODE}" > /tmp/gb_create
parseCreateOutput "$(declare -p METADATA)" ${BLOCKNAME} ${NEWNODE}
ret=$?
echo "" > /tmp/gb_create # clear
if [[ ${ret} -eq 0 ]]; then
echo "${OLDNODE}: CLEANUPSUCCESS" >> "${DIR}/${BLOCKNAME}"
echo "${NEWNODE}: CONFIGSUCCESS" >> "${DIR}/${BLOCKNAME}"
echo "create on '${NEWNODE}' success"
printLog "INFO: create of ${BLOCKNAME} on ${NEWNODE} success"
elif [[ ${ret} -eq 1 ]]; then
echo "create on '${NEWNODE}' skipped"
printLog "WARNING: create of ${BLOCKNAME} on ${NEWNODE} skipped"
else
echo "create on '${NEWNODE}' failed"
printLog "ERROR: create of ${BLOCKNAME} on ${NEWNODE} failed"
printLog "INFO: skipping replace and delete for ${BLOCKNAME}"
echo "-------------------------------------"
# unset variables
unset HOSTS
unset METADATA
releaseLockExit ${fd}
continue;
fi
for NODE in ${HOSTS[@]}; do
if [[ "${NODE}" != "${OLDNODE}" && "${NODE}" != "${NEWNODE}" ]]; then
REPLACEOUT=$(ssh root@${NODE} "$(declare -f runReplace); runReplace '$(declare -p METADATA)' ${OLDNODE} ${NEWNODE}")
parseReplaceOutput "${REPLACEOUT}" ${BLOCKNAME} ${OLDNODE} ${NEWNODE}
ret=$?
if [[ ${ret} -eq 0 ]]; then
echo "replace on '${NODE}' success"
printLog "INFO: replace of ${BLOCKNAME} on ${NODE} success"
elif [[ ${ret} -eq 1 ]]; then
echo "replace on '${NODE}' skipped"
printLog "WARNING: replace of ${BLOCKNAME} on ${NODE} skipped"
else
echo "replace on '${NODE}' failed"
printLog "ERROR: replace of ${BLOCKNAME} on ${NODE} failed"
fi
unset REPLACEOUT
fi
done
DELETEOUT=$(ssh root@${OLDNODE} "$(declare -f runDelete); runDelete '$(declare -p METADATA)' ${BLOCKNAME}")
parseDeleteOutput "${DELETEOUT}" ${BLOCKNAME} ${METADATA["GBID"]}
ret=$?
if [[ ${ret} -eq 0 ]]; then
echo "delete on '${OLDNODE}' success"
printLog "INFO: delete of ${BLOCKNAME} on ${OLDNODE} success"
elif [[ ${ret} -eq 1 ]]; then
echo "delete on '${OLDNODE}' skipped"
printLog "WARNING: delete of ${BLOCKNAME} on ${OLDNODE} skipped"
else
echo "delete on '${OLDNODE}' failed"
printLog "ERROR: delete of ${BLOCKNAME} on ${OLDNODE} failed"
fi
unset DELETEOUT
echo "-------------------------------------"
# unset variables
unset HOSTS
unset METADATA
releaseLockExit ${fd}
done
printLog "Finished replace node job, mount-point=${MOUNTPOINT} old-node=${OLDNODE} new-node=${NEWNODE}"
fdClose ${fd}
|