From 0e8e79350947b9dedd02b1cc2339b8dc42ca599d Mon Sep 17 00:00:00 2001 From: "Kaleb S. KEITHLEY" Date: Thu, 6 Mar 2014 08:13:57 -0500 Subject: gNFS: Set default outstanding RPC limit to 16 Backport of http://review.gluster.org/#/c/6696/ With 64, NFS server hangs with large I/O load (~ 64 threads writing to NFS server). The test results from Ben England (Performance expert) suggest to set it as 16 instead of 64. Change-Id: Iaa9dda512904d2e359d8122a05e5bf65f99a7e78 BUG: 1073441 Signed-off-by: Kaleb S. KEITHLEY Reviewed-on: http://review.gluster.org/7200 Tested-by: Gluster Build System Reviewed-by: Santosh Pradhan --- rpc/rpc-lib/src/rpcsvc.c | 46 ++++++++++++++++++++---------------- rpc/rpc-lib/src/rpcsvc.h | 5 ++-- xlators/nfs/server/src/nfs.c | 22 ++++++++++++++++- xlators/protocol/server/src/server.c | 20 ++++++++++++++-- 4 files changed, 67 insertions(+), 26 deletions(-) diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index 037c157f2..37c3f9663 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1889,8 +1889,7 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options) if (!svc->register_portmap) gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration " "disabled"); - - ret = rpcsvc_set_outstanding_rpc_limit (svc, options); + ret = 0; out: return ret; } @@ -2022,10 +2021,15 @@ out: } /* - * Reconfigure() the rpc.outstanding-rpc-limit param. + * Configure() the rpc.outstanding-rpc-limit param. + * If dict_get_int32() for dict-key "rpc.outstanding-rpc-limit" FAILS, + * it would set the value as "defvalue". Otherwise it would fetch the + * value and round up to multiple-of-8. defvalue must be +ve. + * + * NB: defval or set-value "0" is special which means unlimited/65536. */ int -rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options) +rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue) { int ret = -1; /* FAILURE */ int rpclim = 0; @@ -2034,30 +2038,30 @@ rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options) if ((!svc) || (!options)) return (-1); - /* Reconfigure() the rpc.outstanding-rpc-limit param */ + if ((defvalue < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) || + (defvalue > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT)) { + return (-1); + } + + /* Fetch the rpc.outstanding-rpc-limit from dict. */ ret = dict_get_int32 (options, rpclimkey, &rpclim); if (ret < 0) { /* Fall back to default for FAILURE */ - rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT; - } else { - /* SUCCESS: round off to multiple of 8. - * If the input value fails Boundary check, fall back to - * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT. - * NB: value 0 is special, means its unset i.e. unlimited. - */ - rpclim = ((rpclim + 8 - 1) >> 3) * 8; - if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) { - rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT; - } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) { - rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT; - } + rpclim = defvalue; + } + + /* Round up to multiple-of-8. It must not exceed + * RPCSVC_MAX_OUTSTANDING_RPC_LIMIT. + */ + rpclim = ((rpclim + 8 - 1) >> 3) * 8; + if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) { + rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT; } if (svc->outstanding_rpc_limit != rpclim) { svc->outstanding_rpc_limit = rpclim; gf_log (GF_RPCSVC, GF_LOG_INFO, - "Configured %s with value %d", - rpclimkey, rpclim); + "Configured %s with value %d", rpclimkey, rpclim); } return (0); @@ -2072,7 +2076,7 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options, rpcsvc_t *svc = NULL; int ret = -1; - if ((!ctx) || (!options)) + if ((!xl) || (!ctx) || (!options)) return NULL; svc = GF_CALLOC (1, sizeof (*svc), gf_common_mt_rpcsvc_t); diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h index cbc1f4226..1ccaf0075 100644 --- a/rpc/rpc-lib/src/rpcsvc.h +++ b/rpc/rpc-lib/src/rpcsvc.h @@ -38,7 +38,8 @@ #define MAX_IOVEC 16 #endif -#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 +#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 /* Default for protocol/server */ +#define RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT 16 /* Default for nfs/server */ #define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536 #define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */ @@ -597,7 +598,7 @@ rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options); int rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options); int -rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options); +rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue); int rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen); rpcsvc_vector_sizer diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c index 8c895c66d..bd62b5d4c 100644 --- a/xlators/nfs/server/src/nfs.c +++ b/xlators/nfs/server/src/nfs.c @@ -967,6 +967,15 @@ nfs_init_state (xlator_t *this) goto free_foppool; } + ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc, + this->options, + RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT); + if (ret < 0) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Failed to configure outstanding-rpc-limit"); + goto free_foppool; + } + nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc); this->private = (void *)nfs; @@ -1210,6 +1219,17 @@ reconfigure (xlator_t *this, dict_t *options) "rpcsvc reconfigure options failed"); return (-1); } + + /* Reconfigure rpc.outstanding-rpc-limit */ + ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc, + options, + RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT); + if (ret < 0) { + gf_log (GF_NFS, GF_LOG_ERROR, + "Failed to reconfigure outstanding-rpc-limit"); + return (-1); + } + regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc); if (nfs->register_portmap != regpmap) { nfs->register_portmap = regpmap; @@ -1741,7 +1761,7 @@ struct volume_options options[] = { .type = GF_OPTION_TYPE_INT, .min = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT, .max = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT, - .default_value = TOSTRING(RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT), + .default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT), .description = "Parameter to throttle the number of incoming RPC " "requests from a client. 0 means no limit (can " "potentially run out of memory)" diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 1e23630c2..a8597ca19 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -746,7 +746,15 @@ reconfigure (xlator_t *this, dict_t *options) (void) rpcsvc_set_allow_insecure (rpc_conf, options); (void) rpcsvc_set_root_squash (rpc_conf, options); - (void) rpcsvc_set_outstanding_rpc_limit (rpc_conf, options); + + ret = rpcsvc_set_outstanding_rpc_limit (rpc_conf, options, + RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to reconfigure outstanding-rpc-limit"); + goto out; + } + list_for_each_entry (listeners, &(rpc_conf->listeners), list) { if (listeners->trans != NULL) { if (listeners->trans->reconfigure ) @@ -860,12 +868,20 @@ init (xlator_t *this) /* RPC related */ conf->rpc = rpcsvc_init (this, this->ctx, this->options, 0); if (conf->rpc == NULL) { - gf_log (this->name, GF_LOG_WARNING, + gf_log (this->name, GF_LOG_ERROR, "creation of rpcsvc failed"); ret = -1; goto out; } + ret = rpcsvc_set_outstanding_rpc_limit (conf->rpc, this->options, + RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to configure outstanding-rpc-limit"); + goto out; + } + ret = rpcsvc_create_listeners (conf->rpc, this->options, this->name); if (ret < 1) { -- cgit