From 9c1d7cebbc62723f719b2fd0c45e0a7452e0d6be Mon Sep 17 00:00:00 2001 From: shishir gowda Date: Thu, 4 Apr 2013 11:23:08 +0530 Subject: cluster/distribute: Ignore non-participating subvols for layout checks When subvols-per-directory is < available subvols, then there are layouts which are not populated. This leads to incorrect identification of holes or overlaps. We need to ignore layouts, which have err == 0, and start == stop. In the current scenario (start == stop == 0). Additionally, in layout-merge, treat missing xattrs as err = 0. In case of missing layouts, anomalies will reset them. For any other valid subvoles, err != 0 in case of layouts being zeroed out. Also reverted back dht_selfheal_dir_xattr, which does layout calculation only on subvols which have errors. Change-Id: I9f57062722c9e8a26285e10675c31a78921115a1 BUG: 921408 Signed-off-by: shishir gowda Reviewed-on: http://review.gluster.org/4668 Tested-by: Gluster Build System Reviewed-by: Amar Tumballi Reviewed-by: Jeff Darcy --- tests/bugs/bug-921408.t | 89 ++++++++++++++++++++++++++++++++++ tests/dht.rc | 8 +-- xlators/cluster/dht/src/dht-layout.c | 40 ++++++++------- xlators/cluster/dht/src/dht-selfheal.c | 68 ++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 27 deletions(-) create mode 100755 tests/bugs/bug-921408.t diff --git a/tests/bugs/bug-921408.t b/tests/bugs/bug-921408.t new file mode 100755 index 00000000000..ef2b4fb21cd --- /dev/null +++ b/tests/bugs/bug-921408.t @@ -0,0 +1,89 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../fileio.rc +. $(dirname $0)/../dht.rc + +cleanup; +wait_check_status () +{ + n=0 + while [ $n -lt $1 ] + do + ret=$(rebalance_completed) + if [ $ret == "0" ] + then + return 0; + else + sleep 1 + n=`expr $n + 1`; + fi + done + return 1; +} + +addbr_rebal_till_layout_change() +{ + val=1 + l=$1 + i=1 + while [ $i -lt 5 ] + do + $CLI volume add-brick $V0 $H0:$B0/${V0}$l &>/dev/null + $CLI volume rebalance $V0 fix-layout start &>/dev/null + wait_check_status 15 + if [ $? -eq 1 ] + then + break + fi + NEW_LAYOUT=`get_layout $B0/${V0}0` + if [ $OLD_LAYOUT == $NEW_LAYOUT ] + then + i=`expr $i + 1`; + l=`expr $l + 1`; + else + val=0 + break + fi + done + return $val +} +TEST glusterd +TEST pidof glusterd + +TEST $CLI volume create $V0 $H0:$B0/${V0}0 +TEST $CLI volume set $V0 subvols-per-directory 1 +TEST $CLI volume start $V0 + +TEST glusterfs -s $H0 --volfile-id $V0 $M0; + +TEST mkdir $M0/test +TEST touch $M0/test/test + +fd=`fd_available` +TEST fd_open $fd "rw" $M0/test/test + +OLD_LAYOUT=`get_layout $B0/${V0}0` + +addbr_rebal_till_layout_change 1 + +TEST [ $? -eq 0 ] + +for i in $(seq 1 1000) +do + ls -l $M0/ >/dev/null + ret=$? + if [ $ret != 0 ] + then + break + fi +done + +TEST [ $ret == 0 ]; +TEST fd_close $fd; + +TEST umount $M0 +TEST $CLI volume stop $V0 +TEST $CLI volume delete $V0 + +cleanup diff --git a/tests/dht.rc b/tests/dht.rc index ee92a47bd62..663ea543173 100644 --- a/tests/dht.rc +++ b/tests/dht.rc @@ -2,13 +2,7 @@ function get_layout() { - layout=`getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2` - if [ $? -eq 1] - then - return -1 - else - return $layout - fi + getfattr -n trusted.glusterfs.dht -e hex $1 2>&1|grep dht |cut -d = -f2 } diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c index 7d53e3d04e0..0572298699c 100644 --- a/xlators/cluster/dht/src/dht-layout.c +++ b/xlators/cluster/dht/src/dht-layout.c @@ -366,7 +366,7 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol, } if (ret != 0) { - layout->list[i].err = -1; + layout->list[i].err = 0; gf_log (this->name, GF_LOG_TRACE, "missing disk layout on %s. err = %d", subvol->name, err); @@ -530,23 +530,29 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout, prev_stop = last_stop; for (i = 0; i < layout->cnt; i++) { - if (layout->list[i].err) { - switch (layout->list[i].err) { - case -1: - case ENOENT: - missing++; - break; - case ENOTCONN: - down++; - break; - case ENOSPC: - no_space++; - break; - default: - misc++; + switch (layout->list[i].err) { + case -1: + case ENOENT: + missing++; + break; + case ENOTCONN: + down++; + break; + case ENOSPC: + no_space++; + break; + case 0: + /* if err == 0 and start == stop, then it is a non misc++; + * participating subvolume(spread-cnt). Then, do not + * check for anomalies. If start != stop, then treat it + * as misc err */ + if (layout->list[i].start == layout->list[i].stop) { + continue; } - continue; - } + break; + default: + misc++; + } is_virgin = 0; diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c index e1e962ab92e..037b26fedb4 100644 --- a/xlators/cluster/dht/src/dht-selfheal.c +++ b/xlators/cluster/dht/src/dht-selfheal.c @@ -35,6 +35,7 @@ layout->list[cnt].stop = 0; \ } \ } while (0) + static uint32_t dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n) { @@ -238,6 +239,67 @@ out: return 0; } +int +dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout) +{ + dht_local_t *local = NULL; + int missing_xattr = 0; + int i = 0; + xlator_t *this = NULL; + dht_conf_t *conf = NULL; + dht_layout_t *dummy = NULL; + + local = frame->local; + this = frame->this; + conf = this->private; + + for (i = 0; i < layout->cnt; i++) { + if (layout->list[i].err != -1 || !layout->list[i].stop) { + /* err != -1 would mean xattr present on the directory + * or the directory is non existent. + * !layout->list[i].stop would mean layout absent + */ + + continue; + } + missing_xattr++; + } + + gf_log (this->name, GF_LOG_TRACE, + "%d subvolumes missing xattr for %s", + missing_xattr, loc->path); + + if (missing_xattr == 0) { + dht_selfheal_dir_finish (frame, this, 0); + return 0; + } + + local->call_cnt = missing_xattr; + + for (i = 0; i < layout->cnt; i++) { + if (layout->list[i].err != -1 || !layout->list[i].stop) + continue; + + dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL); + + if (--missing_xattr == 0) + break; + } + dummy = dht_layout_new (this, 1); + if (!dummy) + goto out; + for (i = 0; i < conf->subvolume_cnt; i++) { + if (_gf_false == + dht_is_subvol_in_layout (layout, conf->subvolumes[i])) { + dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0, + conf->subvolumes[i]); + } + } + dht_layout_unref (this, dummy); +out: + return 0; +} + int dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret, int op_errno, struct iatt *statpre, @@ -253,7 +315,7 @@ dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, this_call_cnt = dht_frame_return (frame); if (is_last_call (this_call_cnt)) { - dht_fix_dir_xattr (frame, &local->loc, layout); + dht_selfheal_dir_xattr (frame, &local->loc, layout); } return 0; @@ -278,7 +340,7 @@ dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf, } if (missing_attr == 0) { - dht_fix_dir_xattr (frame, loc, layout); + dht_selfheal_dir_xattr (frame, loc, layout); return 0; } @@ -782,7 +844,7 @@ dht_selfheal_new_directory (call_frame_t *frame, dht_layout_sort_volname (layout); dht_selfheal_layout_new_directory (frame, &local->loc, layout); - dht_fix_dir_xattr (frame, &local->loc, layout); + dht_selfheal_dir_xattr (frame, &local->loc, layout); return 0; } -- cgit