diff options
| author | Richard Wareing <rwareing@fb.com> | 2016-02-29 18:21:43 -0800 |
|---|---|---|
| committer | Shreyas Siravara <sshreyas@fb.com> | 2016-12-20 11:07:39 -0800 |
| commit | f3e876169ea36f639529de0d3a8286d3644f8ef8 (patch) | |
| tree | cd1203f6ab04b3a5509da910dfa58c918f7c2da2 | |
| parent | 8290d5f540d48ae75869ead3511955cd0bda483f (diff) | |
cluster/dht: Bug fixes to cluster.min-free-disk
Summary:
- Enforces FUSE/gNFSd/SHD/rebalance rejection of writes when all subvolumes are
beyond the value set in "cluster.min-free-disk"
- Fixes existing code paths to be more intuitive & straightforward
- Write path now honors min-free-disk
- Adds test to ensure feature doesn't break in future
- This is a port of D2981282 to 3.8
Signed-off-by: Shreyas Siravara <sshreyas@fb.com>
Change-Id: I76923bf76178fe589aa1a26bd1970cf8d009642a
Reviewed-on: http://review.gluster.org/16153
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Shreyas Siravara <sshreyas@fb.com>
Tested-by: Shreyas Siravara <sshreyas@fb.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
| -rwxr-xr-x | tests/basic/dht-min-free-space.t | 78 | ||||
| -rw-r--r-- | tests/basic/gfproxy.t | 2 | ||||
| -rw-r--r-- | tests/bugs/distribute/bug-1099890.t | 2 | ||||
| -rw-r--r-- | tests/bugs/fuse/bug-858488-min-free-disk.t | 1 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-common.c | 41 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-common.h | 6 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-diskusage.c | 53 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-inode-read.c | 10 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-inode-write.c | 13 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/dht-shared.c | 32 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/nufa.c | 10 | ||||
| -rw-r--r-- | xlators/cluster/dht/src/switch.c | 10 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volume-set.c | 12 |
13 files changed, 233 insertions, 37 deletions
diff --git a/tests/basic/dht-min-free-space.t b/tests/basic/dht-min-free-space.t new file mode 100755 index 00000000000..17d10cc39a5 --- /dev/null +++ b/tests/basic/dht-min-free-space.t @@ -0,0 +1,78 @@ +#!/bin/bash + +. $(dirname $0)/../include.rc +. $(dirname $0)/../volume.rc + +cleanup; + +grep $B0/patchy1 /proc/mounts &> /dev/null && umount $B0/patchy1 +grep $B0/patchy2 /proc/mounts &> /dev/null && umount $B0/patchy2 +losetup -d /dev/loop0 2> /dev/null +losetup -d /dev/loop1 2> /dev/null +mkdir $B0/${V0}{1..2} + +TEST glusterd + +TEST dd if=/dev/zero of=/tmp/${V0}-dev1 bs=1M count=30 +TEST dd if=/dev/zero of=/tmp/${V0}-dev2 bs=1M count=30 + +TEST losetup /dev/loop0 /tmp/${V0}-dev1 +TEST losetup /dev/loop1 /tmp/${V0}-dev2 + +TEST mkfs.xfs /dev/loop0 +TEST mkfs.xfs /dev/loop1 + +TEST mount /dev/loop0 $B0/${V0}1 +TEST mount /dev/loop1 $B0/${V0}2 + +TEST $CLI volume create $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}2 +TEST $CLI volume set $V0 cluster.min-free-disk 2MB +TEST $CLI volume set $V0 cluster.min-free-strict-mode on +TEST $CLI volume set $V0 cluster.du-refresh-interval-sec 0 +TEST $CLI volume start $V0 + +TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0 + +#################################### +# Test re-directs of file creation # +#################################### + +# This should work, no redirects +TEST dd if=/dev/zero of=$M0/testfile1 bs=1M count=8 +TEST [ -f /d/backends/${V0}2/testfile1 ] && [ ! -k /d/backends/${V0}1/testfile1 ] + +TEST $CLI volume set $V0 cluster.min-free-disk 19MB + +# This should work, & the file redirected +# Subvolume 2 should have the linkto & +# Subvolume 1 should have the original +TEST dd if=/dev/zero of=$M0/testfile3 bs=1M count=4 +TEST [ -f /d/backends/${V0}1/testfile3 ] && [ ! -k /d/backends/${V0}1/testfile3 ] +TEST [ -k /d/backends/${V0}2/testfile3 ] + +# This should fail, cluster is full +TEST ! dd if=/dev/zero of=$M0/testfile2 bs=1M count=23 + +################### +# Strict mode off # +################### +TEST $CLI volume set $V0 cluster.min-free-strict-mode off +TEST dd if=/dev/zero of=$M0/testfile1 bs=1M count=20 +TEST rm -f $M0/testfile1 + +################### +# Strict mode on # +################### +TEST $CLI volume set $V0 cluster.min-free-strict-mode on +TEST ! dd if=/dev/zero of=$M0/testfile1 bs=1M count=16 +TEST rm -f $M0/testfile1 + +killall gluster{fs,fsd,d} + +umount -lf $B0/${V0}1 +umount -lf $B0/${V0}2 + +losetup -d /dev/loop0 +losetup -d /dev/loop1 + +cleanup; diff --git a/tests/basic/gfproxy.t b/tests/basic/gfproxy.t index 58794394807..71c6788db76 100644 --- a/tests/basic/gfproxy.t +++ b/tests/basic/gfproxy.t @@ -62,7 +62,7 @@ TEST [ "$(md5sum $M0/testfile1 | awk '{print $1}')" == "$md5" ] rm /tmp/testfile1 -dd if=/dev/zero of=$N0/bigfile bs=1M count=10240 & +dd if=/dev/zero of=$N0/bigfile bs=1M count=3072 & BG_STRESS_PID=$! sleep 3 diff --git a/tests/bugs/distribute/bug-1099890.t b/tests/bugs/distribute/bug-1099890.t index 40f70d4938b..29ceccf2309 100644 --- a/tests/bugs/distribute/bug-1099890.t +++ b/tests/bugs/distribute/bug-1099890.t @@ -44,6 +44,8 @@ TEST $CLI volume set $V0 features.quota-deem-statfs on TEST $CLI volume quota $V0 limit-usage / 150MB; +TEST $CLI volume set $V0 cluster.du-refresh-interval-sec 1 + TEST $CLI volume set $V0 cluster.min-free-disk 50% TEST glusterfs -s $H0 --volfile-id=$V0 $M0 diff --git a/tests/bugs/fuse/bug-858488-min-free-disk.t b/tests/bugs/fuse/bug-858488-min-free-disk.t index 635dc04d1e6..ab636575d3f 100644 --- a/tests/bugs/fuse/bug-858488-min-free-disk.t +++ b/tests/bugs/fuse/bug-858488-min-free-disk.t @@ -23,6 +23,7 @@ TEST MOUNT_LOOP $LO2 $B0/${V0}2 ## Lets create volume TEST $CLI volume create $V0 $H0:$B0/${V0}{1,2}; +TEST $CLI volume set $V0 cluster.du-refresh-interval-sec 1 ## Verify volume is created EXPECT "$V0" volinfo_field $V0 'Volume Name'; diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c index 90db73f1f72..50a948ffe30 100644 --- a/xlators/cluster/dht/src/dht-common.c +++ b/xlators/cluster/dht/src/dht-common.c @@ -5550,6 +5550,7 @@ dht_mknod_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, { dht_local_t *local = NULL; xlator_t *avail_subvol = NULL; + int op_errno = 0; local = frame->local; @@ -5562,9 +5563,15 @@ dht_mknod_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, subvol, subvol->fops->mknod, loc, mode, rdev, umask, params); } else { - avail_subvol = dht_free_disk_available_subvol (this, subvol, local); - - if (avail_subvol != subvol) { + /* This will return NULL if all subvolumes are full + * and/or no subvolume needs the min_free_disk limit + */ + avail_subvol = dht_free_disk_available_subvol (this, subvol, + local); + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (avail_subvol != subvol) { local->params = dict_ref (params); local->rdev = rdev; local->mode = mode; @@ -5594,6 +5601,8 @@ dht_mknod_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, } out: return 0; +err: + return op_errno; } int32_t @@ -6233,8 +6242,12 @@ dht_mknod (call_frame_t *frame, xlator_t *this, } } - dht_mknod_wind_to_avail_subvol (frame, this, subvol, loc, rdev, mode, - umask, params); + op_errno = dht_mknod_wind_to_avail_subvol (frame, this, subvol, loc, + rdev, mode, umask, + params); + if (op_errno != 0) { + goto err; + } done: return 0; @@ -6729,6 +6742,7 @@ dht_create_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, { dht_local_t *local = NULL; xlator_t *avail_subvol = NULL; + int op_errno = 0; local = frame->local; @@ -6743,8 +6757,10 @@ dht_create_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, } else { avail_subvol = dht_free_disk_available_subvol (this, subvol, local); - - if (avail_subvol != subvol) { + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (avail_subvol != subvol) { local->params = dict_ref (params); local->flags = flags; local->mode = mode; @@ -6771,6 +6787,10 @@ dht_create_wind_to_avail_subvol (call_frame_t *frame, xlator_t *this, } out: return 0; +err: + DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, + NULL, NULL, NULL); + return op_errno; } int @@ -6873,9 +6893,10 @@ dht_create_do (call_frame_t *frame) goto err; } - dht_create_wind_to_avail_subvol (frame, this, subvol, &local->loc, - local->flags, local->mode, - local->umask, local->fd, local->params); + dht_create_wind_to_avail_subvol (frame, this, subvol, + &local->loc, local->flags, + local->mode, local->umask, + local->fd, local->params); return 0; err: local->refresh_layout_unlock (frame, this, -1, 1); diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h index 29ac798b92b..71b093b20ea 100644 --- a/xlators/cluster/dht/src/dht-common.h +++ b/xlators/cluster/dht/src/dht-common.h @@ -300,6 +300,7 @@ struct dht_du { uint64_t avail_space; uint32_t log; uint32_t chunks; + gf_boolean_t is_full; }; typedef struct dht_du dht_du_t; @@ -484,6 +485,7 @@ struct dht_conf { dht_du_t *du_stats; double min_free_disk; double min_free_inodes; + gf_boolean_t min_free_strict_mode; char disk_unit; int32_t refresh_interval; gf_boolean_t unhashed_sticky_bit; @@ -548,6 +550,10 @@ struct dht_conf { /* lock migration */ gf_boolean_t lock_migration_enabled; + + /* du stats */ + uint32_t du_refresh_interval_sec; + gf_lock_t du_refresh_lock; }; typedef struct dht_conf dht_conf_t; diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c index 1eb9e63c531..1b20dabc61f 100644 --- a/xlators/cluster/dht/src/dht-diskusage.c +++ b/xlators/cluster/dht/src/dht-diskusage.c @@ -153,19 +153,25 @@ dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc) call_frame_t *statfs_frame = NULL; dht_local_t *statfs_local = NULL; struct timeval tv = {0,}; + struct timeval cmp_tv = {0,}; loc_t tmp_loc = {0,}; conf = this->private; + /* Somebody else is already refreshing the statfs info */ + if (TRY_LOCK (&conf->du_refresh_lock) != 0) + return 0; + gettimeofday (&tv, NULL); + cmp_tv = conf->last_stat_fetch; + cmp_tv.tv_sec += conf->du_refresh_interval_sec; + /* make it root gfid, should be enough to get the proper info back */ tmp_loc.gfid[15] = 1; - if (tv.tv_sec > (conf->refresh_interval - + conf->last_stat_fetch.tv_sec)) { - + if (timercmp (&tv, &cmp_tv, >)) { statfs_frame = copy_frame (frame); if (!statfs_frame) { goto err; @@ -200,14 +206,18 @@ dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc) &tmp_loc, statfs_local->params); } - conf->last_stat_fetch.tv_sec = tv.tv_sec; + conf->last_stat_fetch = tv; } - return 0; + ret = 0; + goto out; err: if (statfs_frame) DHT_STACK_DESTROY (statfs_frame); - return -1; + ret = -1; +out: + UNLOCK (&conf->du_refresh_lock); + return ret; } @@ -223,8 +233,13 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol) conf = this->private; /* Check for values above specified percent or free disk */ - LOCK (&conf->subvolume_lock); - { + if (TRY_LOCK (&conf->subvolume_lock) != 0) { + for (i = 0; i < conf->subvolume_cnt; i++) { + if (subvol == conf->subvolumes[i]) { + return conf->du_stats[i].is_full; + } + } + } else { for (i = 0; i < conf->subvolume_cnt; i++) { if (subvol == conf->subvolumes[i]) { if (conf->disk_unit == 'p') { @@ -248,7 +263,15 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol) } } } - } + + /* i will be less than subvolume_cnt if either of + * these booleans are true */ + is_subvol_filled = ( + subvol_filled_space || subvol_filled_inodes); + if (is_subvol_filled) { + conf->du_stats[i].is_full = is_subvol_filled; + } + } UNLOCK (&conf->subvolume_lock); if (subvol_filled_space && conf->subvolume_status[i]) { @@ -273,8 +296,6 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol) } } - is_subvol_filled = (subvol_filled_space || subvol_filled_inodes); - return is_subvol_filled; } @@ -309,15 +330,8 @@ dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol, LOCK (&conf->subvolume_lock); { - avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, + avail_subvol = dht_subvol_maxspace_nonzeroinode(this, subvol, layout); - if(!avail_subvol) - { - avail_subvol = dht_subvol_maxspace_nonzeroinode(this, - subvol, - layout); - } - } UNLOCK (&conf->subvolume_lock); out: @@ -325,7 +339,6 @@ out: gf_msg_debug (this->name, 0, "No subvolume has enough free space \ and/or inodes to create"); - avail_subvol = subvol; } if (layout) diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c index 549f1b9ea7e..e320109c796 100644 --- a/xlators/cluster/dht/src/dht-inode-read.c +++ b/xlators/cluster/dht/src/dht-inode-read.c @@ -104,10 +104,15 @@ dht_open (call_frame_t *frame, xlator_t *this, xlator_t *subvol = NULL; int op_errno = -1; dht_local_t *local = NULL; + dht_conf_t *conf = NULL; VALIDATE_OR_GOTO (frame, err); VALIDATE_OR_GOTO (this, err); VALIDATE_OR_GOTO (fd, err); + conf = this->private; + + if (conf->min_free_strict_mode == _gf_true) + dht_get_du_info (frame, this, loc); local = dht_local_init (frame, loc, fd, GF_FOP_OPEN); if (!local) { @@ -121,6 +126,11 @@ dht_open (call_frame_t *frame, xlator_t *this, "no cached subvolume for fd=%p", fd); op_errno = EINVAL; goto err; + } else if (conf->min_free_strict_mode == _gf_true && + dht_is_subvol_filled (this, subvol) == _gf_true && + flags & O_APPEND) { + op_errno = ENOSPC; + goto err; } local->rebalance.flags = flags; diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c index 112685b659e..7420461da76 100644 --- a/xlators/cluster/dht/src/dht-inode-write.c +++ b/xlators/cluster/dht/src/dht-inode-write.c @@ -161,11 +161,16 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, xlator_t *subvol = NULL; int op_errno = -1; dht_local_t *local = NULL; + loc_t *nil_loc = {0,}; + dht_conf_t *conf = NULL; VALIDATE_OR_GOTO (frame, err); VALIDATE_OR_GOTO (this, err); VALIDATE_OR_GOTO (fd, err); + conf = this->private; + + local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE); if (!local) { @@ -173,15 +178,21 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, goto err; } + if (conf->min_free_strict_mode == _gf_true) + dht_get_du_info (frame, this, nil_loc); + subvol = local->cached_subvol; if (!subvol) { gf_msg_debug (this->name, 0, "no cached subvolume for fd=%p", fd); op_errno = EINVAL; goto err; + } else if (conf->min_free_strict_mode == _gf_true && + dht_is_subvol_filled (this, subvol) == _gf_true) { + op_errno = ENOSPC; + goto err; } - local->rebalance.vector = iov_dup (vector, count); local->rebalance.offset = off; local->rebalance.count = count; diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c index 0fea1d58e58..56bfdedc642 100644 --- a/xlators/cluster/dht/src/dht-shared.c +++ b/xlators/cluster/dht/src/dht-shared.c @@ -439,6 +439,8 @@ dht_reconfigure (xlator_t *this, dict_t *options) conf->disk_unit = 0; if (conf->min_free_disk < 100.0) conf->disk_unit = 'p'; + GF_OPTION_RECONF ("min-free-strict-mode", conf->min_free_strict_mode, + options, bool, out); GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options, percent, out); @@ -495,6 +497,9 @@ dht_reconfigure (xlator_t *this, dict_t *options) GF_OPTION_RECONF ("use-readdirp", conf->use_readdirp, options, bool, out); + + GF_OPTION_RECONF ("du-refresh-interval-sec", + conf->du_refresh_interval_sec, options, uint32, out); ret = 0; out: return ret; @@ -712,7 +717,10 @@ dht_init (xlator_t *this) GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err); GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size, - err); + err); + + GF_OPTION_INIT ("min-free-strict-mode", conf->min_free_strict_mode, + bool, err); GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent, err); @@ -730,6 +738,11 @@ dht_init (xlator_t *this) GF_OPTION_INIT ("lock-migration", conf->lock_migration_enabled, bool, err); + GF_OPTION_INIT ("du-refresh-interval-sec", + conf->du_refresh_interval_sec, uint32, err); + + LOCK_INIT (&conf->du_refresh_lock); + if (defrag) { defrag->lock_migration_enabled = conf->lock_migration_enabled; @@ -901,6 +914,14 @@ struct volume_options options[] = { "process starts balancing out the cluster, and logs will appear " "in log files", }, + { .key = {"min-free-strict-mode"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "off", + .description = "When enabled, will reject in-flight writes or " + "append operations to files when the target subvolume falls " + "below min-free-(disk|inodes). When disabled, these are allowed " + "through and only new files will be affected.", + }, { .key = {"min-free-inodes"}, .type = GF_OPTION_TYPE_PERCENT, .default_value = "5%", @@ -1083,5 +1104,14 @@ struct volume_options options[] = { " associated with a file during rebalance" }, + { .key = {"du-refresh-interval-sec"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, + .default_value = "60", + .validate = GF_OPT_VALIDATE_MIN, + .description = "Specifies how many seconds before subvolume statfs " + "info is re-validated." + }, + { .key = {NULL} }, }; diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c index 56e17d6e884..996faffa37f 100644 --- a/xlators/cluster/dht/src/nufa.c +++ b/xlators/cluster/dht/src/nufa.c @@ -325,7 +325,10 @@ nufa_create (call_frame_t *frame, xlator_t *this, local); } - if (subvol != avail_subvol) { + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (subvol != avail_subvol) { /* create a link file instead of actual file */ local->params = dict_ref (params); local->mode = mode; @@ -430,7 +433,10 @@ nufa_mknod (call_frame_t *frame, xlator_t *this, local); } - if (avail_subvol != subvol) { + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (avail_subvol != subvol) { /* Create linkfile first */ local->params = dict_ref (params); diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c index f1e9a399442..8b14ac99b8f 100644 --- a/xlators/cluster/dht/src/switch.c +++ b/xlators/cluster/dht/src/switch.c @@ -440,7 +440,10 @@ switch_create (call_frame_t *frame, xlator_t *this, local); } - if (subvol != avail_subvol) { + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (subvol != avail_subvol) { /* create a link file instead of actual file */ local->mode = mode; local->flags = flags; @@ -540,7 +543,10 @@ switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, local); } - if (avail_subvol != subvol) { + if (!avail_subvol) { + op_errno = ENOSPC; + goto err; + } else if (avail_subvol != subvol) { /* Create linkfile first */ local->params = dict_ref (params); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index 89e63144fad..ab782db9f5d 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -1048,6 +1048,11 @@ struct volopt_map_entry glusterd_volopt_map[] = { .op_version = 1, .flags = OPT_FLAG_CLIENT_OPT }, + { .key = "cluster.min-free-strict-mode", + .voltype = "cluster/distribute", + .op_version = 1, + .flags = OPT_FLAG_CLIENT_OPT + }, { .key = "cluster.min-free-inodes", .voltype = "cluster/distribute", .op_version = 1, @@ -1113,6 +1118,13 @@ struct volopt_map_entry glusterd_volopt_map[] = { .flags = OPT_FLAG_CLIENT_OPT, }, + { .key = "cluster.du-refresh-interval-sec", + .voltype = "cluster/distribute", + .option = "du-refresh-interval-sec", + .op_version = 1, + .flags = OPT_FLAG_CLIENT_OPT + }, + /* NUFA xlator options (Distribute special case) */ { .key = "cluster.nufa", .voltype = "cluster/distribute", |
