summaryrefslogtreecommitdiffstats
path: root/xlators/features/bit-rot
diff options
context:
space:
mode:
authorKotresh HR <khiremat@redhat.com>2016-07-01 15:54:07 +0530
committerJeff Darcy <jdarcy@redhat.com>2016-07-18 05:03:17 -0700
commit713d7d080df51bf19ce29cf5d682bc006e1c3a19 (patch)
tree10a38c8d8845d879767f2246cf7f77bd4fd5aa4d /xlators/features/bit-rot
parent98b95fc97e896c84de462445c48565a73b2144c2 (diff)
features/bitrot: Move throttling code to libglusterfs
Since throttling is a separate feature by itself, move throttling code to libglusterfs. Change-Id: If9b99885ceb46e5b1865a4af18b2a2caecf59972 BUG: 1352019 Signed-off-by: Kotresh HR <khiremat@redhat.com> Reviewed-on: http://review.gluster.org/14846 Smoke: Gluster Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> Reviewed-by: Ravishankar N <ravishankar@redhat.com> Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Diffstat (limited to 'xlators/features/bit-rot')
-rw-r--r--xlators/features/bit-rot/src/bitd/Makefile.am4
-rw-r--r--xlators/features/bit-rot/src/bitd/bit-rot-scrub.c2
-rw-r--r--xlators/features/bit-rot/src/bitd/bit-rot-tbf.c306
-rw-r--r--xlators/features/bit-rot/src/bitd/bit-rot-tbf.h70
-rw-r--r--xlators/features/bit-rot/src/bitd/bit-rot.c30
-rw-r--r--xlators/features/bit-rot/src/bitd/bit-rot.h4
-rw-r--r--xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h4
7 files changed, 29 insertions, 391 deletions
diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am
index a915f2d..dfa29fd 100644
--- a/xlators/features/bit-rot/src/bitd/Makefile.am
+++ b/xlators/features/bit-rot/src/bitd/Makefile.am
@@ -9,12 +9,12 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(CONTRIBDIR)/timer-wheel \
-I$(top_srcdir)/xlators/features/bit-rot/src/stub
-bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-tbf.c bit-rot-ssm.c \
+bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-ssm.c \
bit-rot-scrub-status.c
bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
$(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
-noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-tbf.h bit-rot-bitd-messages.h bit-rot-ssm.h \
+noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-bitd-messages.h bit-rot-ssm.h \
bit-rot-scrub-status.h
AM_CFLAGS = -Wall -DBR_RATE_LIMIT_SIGNER $(GF_CFLAGS)
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index a588564..7678d8e 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -1938,7 +1938,7 @@ br_scrubber_init (xlator_t *this, br_private_t *priv)
struct br_scrubber *fsscrub = NULL;
int ret = 0;
- priv->tbf = br_tbf_init (NULL, 0);
+ priv->tbf = tbf_init (NULL, 0);
if (!priv->tbf)
return -1;
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c b/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c
deleted file mode 100644
index f8b9b75..0000000
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.c
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
- This file is part of GlusterFS.
-
- This file is licensed to you under your choice of the GNU Lesser
- General Public License, version 3 or any later version (LGPLv3 or
- later), or the GNU General Public License, version 2 (GPLv2), in all
- cases as published by the Free Software Foundation.
-*/
-
-/**
- *
- * Basic token bucket implementation for rate limiting. As of now interfaces
- * to throttle disk read request, directory entry scan and hash calculation
- * are available. To throttle a particular request (operation), the call needs
- * to be wrapped in-between throttling APIs, for e.g.
- *
- * TBF_THROTTLE_BEGIN (...); <-- induces "delays" if required
- * {
- * call (...);
- * }
- * TBF_THROTTLE_END (...); <-- not used atm, maybe needed later
- *
- */
-
-#include "mem-pool.h"
-#include "bit-rot-tbf.h"
-#include "bit-rot-stub-mem-types.h"
-
-typedef struct br_tbf_throttle {
- char done;
-
- pthread_mutex_t mutex;
- pthread_cond_t cond;
-
- unsigned long tokens;
-
- struct list_head list;
-} br_tbf_throttle_t;
-
-/**
- * OK. Most implementations of TBF I've come across generate tokens
- * every second (UML, etc..) and some chose sub-second granularity
- * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
- * any logic for choosing generation interval and it seems pretty
- * logical as one could jack up token count per interval w.r.t.
- * generation rate.
- *
- * Value used here is chosen based on a series of test(s) performed
- * to balance object signing time and not maxing out on all available
- * CPU cores. It's obvious to have seconds granularity and jack up
- * token count per interval, thereby achieving close to similar
- * results. Let's stick to this as it seems to be working fine for
- * the set of ops that are throttled.
- */
-#define BR_TBF_TOKENGEN_INTERVAL_USEC 600000
-
-static br_tbf_throttle_t *
-br_tbf_init_throttle (unsigned long tokens_required)
-{
- br_tbf_throttle_t *throttle = NULL;
-
- throttle = GF_CALLOC (1, sizeof (*throttle),
- gf_br_mt_br_tbf_throttle_t);
- if (!throttle)
- return NULL;
-
- throttle->done = 0;
- throttle->tokens = tokens_required;
- INIT_LIST_HEAD (&throttle->list);
-
- (void) pthread_mutex_init (&throttle->mutex, NULL);
- (void) pthread_cond_init (&throttle->cond, NULL);
-
- return throttle;
-}
-
-void
-_br_tbf_dispatch_queued (br_tbf_bucket_t *bucket)
-{
- gf_boolean_t xcont = _gf_false;
- br_tbf_throttle_t *tmp = NULL;
- br_tbf_throttle_t *throttle = NULL;
-
- list_for_each_entry_safe (throttle, tmp, &bucket->queued, list) {
-
- pthread_mutex_lock (&throttle->mutex);
- {
- if (bucket->tokens < throttle->tokens) {
- xcont = _gf_true;
- goto unblock;
- }
-
- /* this request can now be serviced */
- throttle->done = 1;
- list_del_init (&throttle->list);
-
- bucket->tokens -= throttle->tokens;
- pthread_cond_signal (&throttle->cond);
- }
- unblock:
- pthread_mutex_unlock (&throttle->mutex);
- if (xcont)
- break;
- }
-}
-
-void *br_tbf_tokengenerator (void *arg)
-{
- unsigned long tokenrate = 0;
- unsigned long maxtokens = 0;
- br_tbf_bucket_t *bucket = arg;
-
- tokenrate = bucket->tokenrate;
- maxtokens = bucket->maxtokens;
-
- while (1) {
- usleep (BR_TBF_TOKENGEN_INTERVAL_USEC);
-
- LOCK (&bucket->lock);
- {
- bucket->tokens += tokenrate;
- if (bucket->tokens > maxtokens)
- bucket->tokens = maxtokens;
-
- if (!list_empty (&bucket->queued))
- _br_tbf_dispatch_queued (bucket);
- }
- UNLOCK (&bucket->lock);
- }
-
- return NULL;
-}
-
-/**
- * There is lazy synchronization between this routine (when invoked
- * under br_tbf_mod() context) and br_tbf_throttle(). *bucket is
- * updated _after_ all the required variables are initialized.
- */
-static int32_t
-br_tbf_init_bucket (br_tbf_t *tbf, br_tbf_opspec_t *spec)
-{
- int ret = 0;
- br_tbf_bucket_t *curr = NULL;
- br_tbf_bucket_t **bucket = NULL;
-
- GF_ASSERT (spec->op >= BR_TBF_OP_MIN);
- GF_ASSERT (spec->op <= BR_TBF_OP_MAX);
-
- /* no rate? no throttling. */
- if (!spec->rate)
- return 0;
-
- bucket = tbf->bucket + spec->op;
-
- curr = GF_CALLOC (1, sizeof (*curr), gf_br_mt_br_tbf_bucket_t);
- if (!curr)
- goto error_return;
-
- LOCK_INIT (&curr->lock);
- INIT_LIST_HEAD (&curr->queued);
-
- curr->tokens = 0;
- curr->tokenrate = spec->rate;
- curr->maxtokens = spec->maxlimit;
-
- ret = gf_thread_create (&curr->tokener,
- NULL, br_tbf_tokengenerator, curr);
- if (ret != 0)
- goto freemem;
-
- *bucket = curr;
- return 0;
-
- freemem:
- LOCK_DESTROY (&curr->lock);
- GF_FREE (curr);
- error_return:
- return -1;
-}
-
-#define BR_TBF_ALLOC_SIZE \
- (sizeof (br_tbf_t) + (BR_TBF_OP_MAX * sizeof (br_tbf_bucket_t)))
-
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *tbfspec, unsigned int count)
-{
- int32_t i = 0;
- int32_t ret = 0;
- br_tbf_t *tbf = NULL;
- br_tbf_opspec_t *opspec = NULL;
-
- tbf = GF_CALLOC (1, BR_TBF_ALLOC_SIZE, gf_br_mt_br_tbf_t);
- if (!tbf)
- goto error_return;
-
- tbf->bucket = (br_tbf_bucket_t **) ((char *)tbf + sizeof (*tbf));
- for (i = 0; i < BR_TBF_OP_MAX; i++) {
- *(tbf->bucket + i) = NULL;
- }
-
- for (i = 0; i < count; i++) {
- opspec = tbfspec + i;
-
- ret = br_tbf_init_bucket (tbf, opspec);
- if (ret)
- break;
- }
-
- if (ret)
- goto error_return;
-
- return tbf;
-
- error_return:
- return NULL;
-}
-
-static void
-br_tbf_mod_bucket (br_tbf_bucket_t *bucket, br_tbf_opspec_t *spec)
-{
- LOCK (&bucket->lock);
- {
- bucket->tokens = 0;
- bucket->tokenrate = spec->rate;
- bucket->maxtokens = spec->maxlimit;
- }
- UNLOCK (&bucket->lock);
-
- /* next token tick would unqueue pending operations */
-}
-
-int
-br_tbf_mod (br_tbf_t *tbf, br_tbf_opspec_t *tbfspec)
-{
- int ret = 0;
- br_tbf_bucket_t *bucket = NULL;
- br_tbf_ops_t op = BR_TBF_OP_MIN;
-
- if (!tbf || !tbfspec)
- return -1;
-
- op = tbfspec->op;
-
- GF_ASSERT (op >= BR_TBF_OP_MIN);
- GF_ASSERT (op <= BR_TBF_OP_MAX);
-
- bucket = *(tbf->bucket + op);
- if (bucket) {
- br_tbf_mod_bucket (bucket, tbfspec);
- } else {
- ret = br_tbf_init_bucket (tbf, tbfspec);
- }
-
- return ret;
-}
-
-void
-br_tbf_throttle (br_tbf_t *tbf, br_tbf_ops_t op, unsigned long tokens_requested)
-{
- char waitq = 0;
- br_tbf_bucket_t *bucket = NULL;
- br_tbf_throttle_t *throttle = NULL;
-
- GF_ASSERT (op >= BR_TBF_OP_MIN);
- GF_ASSERT (op <= BR_TBF_OP_MAX);
-
- bucket = *(tbf->bucket + op);
- if (!bucket)
- return;
-
- LOCK (&bucket->lock);
- {
- /**
- * if there are enough tokens in the bucket there is no need
- * to throttle the request: therefore, consume the required
- * number of tokens and continue.
- */
- if (tokens_requested <= bucket->tokens) {
- bucket->tokens -= tokens_requested;
- } else {
- throttle = br_tbf_init_throttle (tokens_requested);
- if (!throttle) /* let it slip through for now.. */
- goto unblock;
-
- waitq = 1;
- pthread_mutex_lock (&throttle->mutex);
- list_add_tail (&throttle->list, &bucket->queued);
- }
- }
- unblock:
- UNLOCK (&bucket->lock);
-
- if (waitq) {
- while (!throttle->done) {
- pthread_cond_wait (&throttle->cond, &throttle->mutex);
- }
-
- pthread_mutex_unlock (&throttle->mutex);
-
- pthread_mutex_destroy (&throttle->mutex);
- pthread_cond_destroy (&throttle->cond);
-
- GF_FREE (throttle);
- }
-}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h b/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h
deleted file mode 100644
index 5a41be4..0000000
--- a/xlators/features/bit-rot/src/bitd/bit-rot-tbf.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
- This file is part of GlusterFS.
-
- This file is licensed to you under your choice of the GNU Lesser
- General Public License, version 3 or any later version (LGPLv3 or
- later), or the GNU General Public License, version 2 (GPLv2), in all
- cases as published by the Free Software Foundation.
-*/
-
-#include "list.h"
-#include "xlator.h"
-#include "locking.h"
-
-#ifndef __BIT_ROT_TBF_H__
-#define __BIT_ROT_TBF_H__
-
-typedef enum br_tbf_ops {
- BR_TBF_OP_MIN = -1,
- BR_TBF_OP_HASH = 0, /* checksum calculation */
- BR_TBF_OP_READ = 1, /* inode read(s) */
- BR_TBF_OP_READDIR = 2, /* dentry read(s) */
- BR_TBF_OP_MAX = 3,
-} br_tbf_ops_t;
-
-/**
- * Operation rate specification
- */
-typedef struct br_tbf_opspec {
- br_tbf_ops_t op;
-
- unsigned long rate;
-
- unsigned long maxlimit;
-} br_tbf_opspec_t;
-
-/**
- * Token bucket for each operation type
- */
-typedef struct br_tbf_bucket {
- gf_lock_t lock;
-
- pthread_t tokener; /* token generator thread */
-
- unsigned long tokenrate; /* token generation rate */
-
- unsigned long tokens; /* number of current tokens */
-
- unsigned long maxtokens; /* maximum token in the bucket */
-
- struct list_head queued; /* list of non-conformant requests */
-} br_tbf_bucket_t;
-
-typedef struct br_tbf {
- br_tbf_bucket_t **bucket;
-} br_tbf_t;
-
-br_tbf_t *
-br_tbf_init (br_tbf_opspec_t *, unsigned int);
-
-int
-br_tbf_mod (br_tbf_t *, br_tbf_opspec_t *);
-
-void
-br_tbf_throttle (br_tbf_t *, br_tbf_ops_t, unsigned long);
-
-#define TBF_THROTTLE_BEGIN(tbf, op, tokens) (br_tbf_throttle (tbf, op, tokens))
-#define TBF_THROTTLE_END(tbf, op, tokens) (void)
-
-#endif /** __BIT_ROT_TBF_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index 174af2b..ca3fc27 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -278,7 +278,7 @@ br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
off_t offset, size_t size, SHA256_CTX *sha256)
{
int32_t ret = -1;
- br_tbf_t *tbf = NULL;
+ tbf_t *tbf = NULL;
struct iovec *iovec = NULL;
struct iobref *iobref = NULL;
br_private_t *priv = NULL;
@@ -311,12 +311,12 @@ br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
goto out;
for (i = 0; i < count; i++) {
- TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
+ TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
{
SHA256_Update (sha256, (const unsigned char *)
(iovec[i].iov_base), iovec[i].iov_len);
}
- TBF_THROTTLE_BEGIN (tbf, BR_TBF_OP_HASH, iovec[i].iov_len);
+ TBF_THROTTLE_BEGIN (tbf, TBF_OP_HASH, iovec[i].iov_len);
}
out:
@@ -1756,14 +1756,32 @@ static int32_t
br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
{
br_private_t *priv = NULL;
- br_tbf_opspec_t spec = {0,};
+ tbf_opspec_t spec = {0,};
priv = this->private;
- spec.op = BR_TBF_OP_HASH;
+ spec.op = TBF_OP_HASH;
spec.rate = 0;
spec.maxlimit = 0;
+/**
+ * OK. Most implementations of TBF I've come across generate tokens
+ * every second (UML, etc..) and some chose sub-second granularity
+ * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
+ * any logic for choosing generation interval and it seems pretty
+ * logical as one could jack up token count per interval w.r.t.
+ * generation rate.
+ *
+ * Value used here is chosen based on a series of test(s) performed
+ * to balance object signing time and not maxing out on all available
+ * CPU cores. It's obvious to have seconds granularity and jack up
+ * token count per interval, thereby achieving close to similar
+ * results. Let's stick to this as it seems to be working fine for
+ * the set of ops that are throttled.
+ **/
+ spec.token_gen_interval = 600000; /* In usec */
+
+
#ifdef BR_RATE_LIMIT_SIGNER
double contribution = 0;
@@ -1783,7 +1801,7 @@ br_rate_limit_signer (xlator_t *this, int child_count, int numbricks)
"[Rate Limit Info] \"tokens/sec (rate): %lu, "
"maxlimit: %lu\"", spec.rate, spec.maxlimit);
- priv->tbf = br_tbf_init (&spec, 1);
+ priv->tbf = tbf_init (&spec, 1);
return priv->tbf ? 0 : -1;
}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index 8e9267048..b5448f7 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -21,7 +21,7 @@
#include "changelog.h"
#include "timer-wheel.h"
-#include "bit-rot-tbf.h"
+#include "throttle-tbf.h"
#include "bit-rot-ssm.h"
#include "bit-rot-common.h"
@@ -209,7 +209,7 @@ struct br_private {
uint32_t expiry_time; /* objects "wait" time */
- br_tbf_t *tbf; /* token bucket filter */
+ tbf_t *tbf; /* token bucket filter */
gf_boolean_t iamscrubber; /* function as a fs scrubber */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index f70fafb..a33577c 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -22,10 +22,6 @@ enum br_mem_types {
gf_br_mt_br_child_t,
gf_br_mt_br_object_t,
gf_br_mt_br_ob_n_wk_t,
- gf_br_mt_br_tbf_t,
- gf_br_mt_br_tbf_bucket_t,
- gf_br_mt_br_tbf_throttle_t,
- gf_br_mt_br_tbf_opspec_t,
gf_br_mt_br_scrubber_t,
gf_br_mt_br_fsscan_entry_t,
gf_br_stub_mt_br_stub_fd_t,