diff options
Diffstat (limited to 'xlators')
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot.c | 45 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/bitd/bit-rot.h | 20 | ||||
| -rw-r--r-- | xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h | 1 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-bitrot.c | 85 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volgen.c | 6 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volume-set.c | 9 |
6 files changed, 145 insertions, 21 deletions
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c index abb078b21cf..a2f1c343a1d 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.c +++ b/xlators/features/bit-rot/src/bitd/bit-rot.c @@ -1717,22 +1717,26 @@ out: return 0; } -/** - * Initialize signer specific structures, spawn worker threads. - */ - static void br_fini_signer(xlator_t *this, br_private_t *priv) { int i = 0; - for (; i < BR_WORKERS; i++) { + if (priv == NULL) + return; + + for (; i < priv->signer_th_count; i++) { (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]); } + GF_FREE(priv->obj_queue->workers); pthread_cond_destroy(&priv->object_cond); } +/** + * Initialize signer specific structures, spawn worker threads. + */ + static int32_t br_init_signer(xlator_t *this, br_private_t *priv) { @@ -1752,7 +1756,12 @@ br_init_signer(xlator_t *this, br_private_t *priv) goto cleanup_cond; INIT_LIST_HEAD(&priv->obj_queue->objects); - for (i = 0; i < BR_WORKERS; i++) { + priv->obj_queue->workers = GF_CALLOC( + priv->signer_th_count, sizeof(pthread_t), gf_br_mt_br_worker_t); + if (!priv->obj_queue->workers) + goto cleanup_obj_queue; + + for (i = 0; i < priv->signer_th_count; i++) { ret = gf_thread_create(&priv->obj_queue->workers[i], NULL, br_process_object, this, "brpobj"); if (ret != 0) { @@ -1769,7 +1778,9 @@ cleanup_threads: for (i--; i >= 0; i--) { (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]); } + GF_FREE(priv->obj_queue->workers); +cleanup_obj_queue: GF_FREE(priv->obj_queue); cleanup_cond: @@ -1822,7 +1833,7 @@ br_rate_limit_signer(xlator_t *this, int child_count, int numbricks) if (contribution == 0) contribution = 1; spec.rate = BR_HASH_CALC_READ_SIZE * contribution; - spec.maxlimit = BR_WORKERS * BR_HASH_CALC_READ_SIZE; + spec.maxlimit = priv->signer_th_count * BR_HASH_CALC_READ_SIZE; #endif @@ -1841,11 +1852,16 @@ br_rate_limit_signer(xlator_t *this, int child_count, int numbricks) static int32_t br_signer_handle_options(xlator_t *this, br_private_t *priv, dict_t *options) { - if (options) + if (options) { GF_OPTION_RECONF("expiry-time", priv->expiry_time, options, uint32, error_return); - else + GF_OPTION_RECONF("signer-threads", priv->signer_th_count, options, + uint32, error_return); + } else { GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return); + GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32, + error_return); + } return 0; @@ -1861,6 +1877,8 @@ br_signer_init(xlator_t *this, br_private_t *priv) GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return); GF_OPTION_INIT("brick-count", numbricks, int32, error_return); + GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32, + error_return); ret = br_rate_limit_signer(this, priv->child_count, numbricks); if (ret) @@ -2187,6 +2205,15 @@ struct volume_options options[] = { .description = "Pause/Resume scrub. Upon resume, scrubber " "continues from where it left off.", }, + { + .key = {"signer-threads"}, + .type = GF_OPTION_TYPE_INT, + .default_value = BR_WORKERS, + .op_version = {GD_OP_VERSION_8_0}, + .flags = OPT_FLAG_SETTABLE, + .description = "Number of signing process threads. As a best " + "practice, set this to the number of processor cores", + }, {.key = {NULL}}, }; diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h index a4d4fd74198..8ac7dcdac3d 100644 --- a/xlators/features/bit-rot/src/bitd/bit-rot.h +++ b/xlators/features/bit-rot/src/bitd/bit-rot.h @@ -30,12 +30,6 @@ #include <openssl/sha.h> -/** - * TODO: make this configurable. As a best practice, set this to the - * number of processor cores. - */ -#define BR_WORKERS 4 - typedef enum scrub_throttle { BR_SCRUB_THROTTLE_VOID = -1, BR_SCRUB_THROTTLE_LAZY = 0, @@ -108,12 +102,12 @@ struct br_child { typedef struct br_child br_child_t; struct br_obj_n_workers { - struct list_head objects; /* queue of objects expired from the - timer wheel and ready to be picked - up for signing */ - pthread_t workers[BR_WORKERS]; /* Threads which pick up the objects - from the above queue and start - signing each object */ + struct list_head objects; /* queue of objects expired from the + timer wheel and ready to be picked + up for signing */ + pthread_t *workers; /* Threads which pick up the objects + from the above queue and start + signing each object */ }; struct br_scrubber { @@ -209,6 +203,8 @@ struct br_private { uint32_t expiry_time; /* objects "wait" time */ + uint32_t signer_th_count; /* Number of signing process threads */ + tbf_t *tbf; /* token bucket filter */ gf_boolean_t iamscrubber; /* function as a fs scrubber */ diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h index 40bcda110e6..9d93caf069f 100644 --- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h +++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h @@ -29,6 +29,7 @@ enum br_mem_types { gf_br_stub_mt_sigstub_t, gf_br_mt_br_child_event_t, gf_br_stub_mt_misc, + gf_br_mt_br_worker_t, gf_br_stub_mt_end, }; diff --git a/xlators/mgmt/glusterd/src/glusterd-bitrot.c b/xlators/mgmt/glusterd/src/glusterd-bitrot.c index 9959a59e40c..37429fe9214 100644 --- a/xlators/mgmt/glusterd/src/glusterd-bitrot.c +++ b/xlators/mgmt/glusterd/src/glusterd-bitrot.c @@ -34,6 +34,7 @@ const char *gd_bitrot_op_list[GF_BITROT_OPTION_TYPE_MAX] = { [GF_BITROT_OPTION_TYPE_SCRUB_FREQ] = "scrub-frequency", [GF_BITROT_OPTION_TYPE_SCRUB] = "scrub", [GF_BITROT_OPTION_TYPE_EXPIRY_TIME] = "expiry-time", + [GF_BITROT_OPTION_TYPE_SIGNER_THREADS] = "signer-threads", }; int @@ -354,6 +355,81 @@ out: return ret; } +static gf_boolean_t +is_bitd_configure_noop(xlator_t *this, glusterd_volinfo_t *volinfo) +{ + gf_boolean_t noop = _gf_true; + glusterd_brickinfo_t *brickinfo = NULL; + + if (!glusterd_is_bitrot_enabled(volinfo)) + goto out; + else if (volinfo->status != GLUSTERD_STATUS_STARTED) + goto out; + else { + cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list) + { + if (!glusterd_is_local_brick(this, volinfo, brickinfo)) + continue; + noop = _gf_false; + return noop; + } + } +out: + return noop; +} + +static int +glusterd_bitrot_signer_threads(glusterd_volinfo_t *volinfo, dict_t *dict, + char *key, char **op_errstr) +{ + int32_t ret = -1; + uint32_t signer_th_count = 0; + uint32_t existing_th_count = 0; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + char dkey[32] = { + 0, + }; + + this = THIS; + GF_ASSERT(this); + + priv = this->private; + GF_VALIDATE_OR_GOTO(this->name, priv, out); + + ret = dict_get_uint32(dict, "signer-threads", &signer_th_count); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, + "Unable to get bitrot signer thread count."); + goto out; + } + + ret = dict_get_uint32(volinfo->dict, key, &existing_th_count); + if (ret == 0 && signer_th_count == existing_th_count) { + goto out; + } + + snprintf(dkey, sizeof(dkey), "%d", signer_th_count); + ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED, + "Failed to set option %s", key); + goto out; + } + + if (!is_bitd_configure_noop(this, volinfo)) { + ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL, + PROC_START_NO_WAIT); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL, + "Failed to reconfigure bitrot services"); + goto out; + } + } +out: + return ret; +} + static int glusterd_bitrot_enable(glusterd_volinfo_t *volinfo, char **op_errstr) { @@ -594,6 +670,15 @@ glusterd_op_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict) volinfo, dict, "features.expiry-time", op_errstr); if (ret) goto out; + break; + + case GF_BITROT_OPTION_TYPE_SIGNER_THREADS: + ret = glusterd_bitrot_signer_threads( + volinfo, dict, "features.signer-threads", op_errstr); + if (ret) + goto out; + break; + case GF_BITROT_CMD_SCRUB_STATUS: case GF_BITROT_CMD_SCRUB_ONDEMAND: break; diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c index d9e5e865fbd..aea25b1d206 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volgen.c +++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c @@ -4308,6 +4308,12 @@ bitrot_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme, return -1; } + if (!strcmp(vme->option, "signer-threads")) { + ret = xlator_set_fixed_option(xl, "signer-threads", vme->value); + if (ret) + return -1; + } + return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c index cec5b1a78d3..63de3ef685c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c @@ -2651,6 +2651,15 @@ struct volopt_map_entry glusterd_volopt_map[] = { .op_version = GD_OP_VERSION_3_7_0, .type = NO_DOC, }, + { + .key = "features.signer-threads", + .voltype = "features/bit-rot", + .value = BR_WORKERS, + .option = "signer-threads", + .op_version = GD_OP_VERSION_8_0, + .type = NO_DOC, + }, + /* Upcall translator options */ /* Upcall translator options */ { .key = "features.cache-invalidation", |