gNFS: Set default outstanding RPC limit to 16

Backport of http://review.gluster.org/#/c/6696/

With 64, NFS server hangs with large I/O load (~ 64 threads writing
to NFS server). The test results from Ben England (Performance expert)
suggest to set it as 16 instead of 64.

Change-Id: Iaa9dda512904d2e359d8122a05e5bf65f99a7e78
BUG: 1073441
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
Reviewed-on: http://review.gluster.org/7200
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Santosh Pradhan <spradhan@redhat.com>

2 files changed, 28 insertions, 23 deletions

diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index 037c157f21e..37c3f96630d 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1889,8 +1889,7 @@ rpcsvc_init_options (rpcsvc_t *svc, dict_t *options)
         if (!svc->register_portmap)
                 gf_log (GF_RPCSVC, GF_LOG_DEBUG, "Portmap registration "
                         "disabled");
-
-        ret = rpcsvc_set_outstanding_rpc_limit (svc, options);
+        ret = 0;
 out:
         return ret;
 }
@@ -2022,10 +2021,15 @@ out:
 }
 
 /*
- * Reconfigure() the rpc.outstanding-rpc-limit param.
+ * Configure() the rpc.outstanding-rpc-limit param.
+ * If dict_get_int32() for dict-key "rpc.outstanding-rpc-limit" FAILS,
+ * it would set the value as "defvalue". Otherwise it would fetch the
+ * value and round up to multiple-of-8. defvalue must be +ve.
+ *
+ * NB: defval or set-value "0" is special which means unlimited/65536.
  */
 int
-rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue)
 {
         int            ret        = -1; /* FAILURE */
         int            rpclim     = 0;
@@ -2034,30 +2038,30 @@ rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options)
         if ((!svc) || (!options))
                 return (-1);
 
-        /* Reconfigure() the rpc.outstanding-rpc-limit param */
+        if ((defvalue < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) ||
+            (defvalue > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT)) {
+                return (-1);
+        }
+
+        /* Fetch the rpc.outstanding-rpc-limit from dict. */
         ret = dict_get_int32 (options, rpclimkey, &rpclim);
         if (ret < 0) {
                 /* Fall back to default for FAILURE */
-                rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
-        } else {
-                /* SUCCESS: round off to multiple of 8.
-                 * If the input value fails Boundary check, fall back to
-                 * default i.e. RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT.
-                 * NB: value 0 is special, means its unset i.e. unlimited.
-                 */
-                rpclim = ((rpclim + 8 - 1) >> 3) * 8;
-                if (rpclim < RPCSVC_MIN_OUTSTANDING_RPC_LIMIT) {
-                        rpclim = RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT;
-                } else if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
-                        rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
-                }
+                rpclim = defvalue;
+        }
+
+        /* Round up to multiple-of-8. It must not exceed
+         * RPCSVC_MAX_OUTSTANDING_RPC_LIMIT.
+         */
+        rpclim = ((rpclim + 8 - 1) >> 3) * 8;
+        if (rpclim > RPCSVC_MAX_OUTSTANDING_RPC_LIMIT) {
+                rpclim = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT;
         }
 
         if (svc->outstanding_rpc_limit != rpclim) {
                 svc->outstanding_rpc_limit = rpclim;
                 gf_log (GF_RPCSVC, GF_LOG_INFO,
-                                   "Configured %s with value %d",
-                                   rpclimkey, rpclim);
+                        "Configured %s with value %d", rpclimkey, rpclim);
         }
 
         return (0);
@@ -2072,7 +2076,7 @@ rpcsvc_init (xlator_t *xl, glusterfs_ctx_t *ctx, dict_t *options,
         rpcsvc_t          *svc              = NULL;
         int                ret              = -1;
 
-        if ((!ctx) || (!options))
+        if ((!xl) || (!ctx) || (!options))
                 return NULL;
 
         svc = GF_CALLOC (1, sizeof (*svc), gf_common_mt_rpcsvc_t);
diff --git a/rpc/rpc-lib/src/rpcsvc.h b/rpc/rpc-lib/src/rpcsvc.h
index cbc1f4226b8..1ccaf0075b9 100644
--- a/rpc/rpc-lib/src/rpcsvc.h
+++ b/rpc/rpc-lib/src/rpcsvc.h
@@ -38,7 +38,8 @@
 #define MAX_IOVEC 16
 #endif
 
-#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64
+#define RPCSVC_DEFAULT_OUTSTANDING_RPC_LIMIT 64 /* Default for protocol/server */
+#define RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT 16 /* Default for nfs/server */
 #define RPCSVC_MAX_OUTSTANDING_RPC_LIMIT 65536
 #define RPCSVC_MIN_OUTSTANDING_RPC_LIMIT 0 /* No limit i.e. Unlimited */
 
@@ -597,7 +598,7 @@ rpcsvc_set_addr_namelookup (rpcsvc_t *svc, dict_t *options);
 int
 rpcsvc_set_root_squash (rpcsvc_t *svc, dict_t *options);
 int
-rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options);
+rpcsvc_set_outstanding_rpc_limit (rpcsvc_t *svc, dict_t *options, int defvalue);
 int
 rpcsvc_auth_array (rpcsvc_t *svc, char *volname, int *autharr, int arrlen);
 rpcsvc_vector_sizer