fuse: make background queue length configurable

* also make 'congestion_threshold' an option
* make 'congestion_threshold' as 75% of background queue length if
  not explicitely specified
* in glusterfsd.c, moved all the fuse option dictionary setting
  code to separate function

Change-Id: Ie1680eefaed9377720770a09222282321bd4132e
Signed-off-by: Amar Tumballi <amarts@redhat.com>
BUG: 845214
Reviewed-on: https://code.engineering.redhat.com/gerrit/1860
Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Tested-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 164 insertions, 56 deletions

diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index 9e21204929f..58fa38d0b3b 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -185,6 +185,12 @@ static struct argp_option gf_options[] = {
 	{"gid-timeout", ARGP_GID_TIMEOUT_KEY, "SECONDS", 0,
 	 "Set auxilary group list timeout to SECONDS for fuse translator "
 	 "[default: 0]"},
+	{"background-qlen", ARGP_FUSE_BACKGROUND_QLEN_KEY, "N", 0,
+	 "Set fuse module's background queue length to N "
+	 "[default: 64]"},
+	{"congestion-threshold", ARGP_FUSE_CONGESTION_THRESHOLD_KEY, "N", 0,
+	 "Set fuse module's congestion threshold to N "
+	 "[default: 48]"},
         {"client-pid", ARGP_CLIENT_PID_KEY, "PID", OPTION_HIDDEN,
          "client will authenticate itself with process id PID to server"},
         {"user-map-root", ARGP_USER_MAP_ROOT_KEY, "USER", OPTION_HIDDEN,
@@ -208,50 +214,17 @@ int glusterfs_mgmt_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_init (glusterfs_ctx_t *ctx);
 int glusterfs_listener_stop (glusterfs_ctx_t *ctx);
 
-int
-create_fuse_mount (glusterfs_ctx_t *ctx)
+
+static int
+set_fuse_mount_options (glusterfs_ctx_t *ctx, dict_t *options)
 {
         int              ret = 0;
         cmd_args_t      *cmd_args = NULL;
-        xlator_t        *master = NULL;
         char            *mount_point = NULL;
         char            cwd[PATH_MAX] = {0,};
 
         cmd_args = &ctx->cmd_args;
 
-        if (!cmd_args->mount_point) {
-                gf_log ("", GF_LOG_TRACE,
-                        "mount point not found, not a client process");
-                return 0;
-        }
-
-        if (ctx->process_mode != GF_CLIENT_PROCESS) {
-                gf_log("glusterfsd", GF_LOG_ERROR,
-                       "Not a client process, not performing mount operation");
-                return -1;
-        }
-
-        master = GF_CALLOC (1, sizeof (*master),
-                            gfd_mt_xlator_t);
-        if (!master)
-                goto err;
-
-        master->name = gf_strdup ("fuse");
-        if (!master->name)
-                goto err;
-
-        if (xlator_set_type (master, "mount/fuse") == -1) {
-                gf_log ("glusterfsd", GF_LOG_ERROR,
-                        "MOUNT-POINT %s initialization failed",
-                        cmd_args->mount_point);
-                goto err;
-        }
-
-        master->ctx      = ctx;
-        master->options  = get_new_dict ();
-        if (!master->options)
-                goto err;
-
         /* Check if mount-point is absolute path,
          * if not convert to absolute path by concating with CWD
          */
@@ -273,7 +246,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         } else
                 mount_point = gf_strdup (cmd_args->mount_point);
 
-        ret = dict_set_dynstr (master->options, ZR_MOUNTPOINT_OPT, mount_point);
+        ret = dict_set_dynstr (options, ZR_MOUNTPOINT_OPT, mount_point);
         if (ret < 0) {
                 gf_log ("glusterfsd", GF_LOG_ERROR,
                         "failed to set mount-point to options dictionary");
@@ -281,7 +254,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_attribute_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ATTR_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ATTR_TIMEOUT_OPT,
                                        cmd_args->fuse_attribute_timeout);
 
                 if (ret < 0) {
@@ -293,7 +266,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->fuse_entry_timeout >= 0) {
-                ret = dict_set_double (master->options, ZR_ENTRY_TIMEOUT_OPT,
+                ret = dict_set_double (options, ZR_ENTRY_TIMEOUT_OPT,
                                        cmd_args->fuse_entry_timeout);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -304,7 +277,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->client_pid_set) {
-                ret = dict_set_int32 (master->options, "client-pid",
+                ret = dict_set_int32 (options, "client-pid",
                                       cmd_args->client_pid);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -315,7 +288,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->uid_map_root) {
-                ret = dict_set_int32 (master->options, "uid-map-root",
+                ret = dict_set_int32 (options, "uid-map-root",
                                       cmd_args->uid_map_root);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -326,7 +299,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->volfile_check) {
-                ret = dict_set_int32 (master->options, ZR_STRICT_VOLFILE_CHECK,
+                ret = dict_set_int32 (options, ZR_STRICT_VOLFILE_CHECK,
                                       cmd_args->volfile_check);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -337,7 +310,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->dump_fuse) {
-                ret = dict_set_static_ptr (master->options, ZR_DUMP_FUSE,
+                ret = dict_set_static_ptr (options, ZR_DUMP_FUSE,
                                            cmd_args->dump_fuse);
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -348,7 +321,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->acl) {
-                ret = dict_set_static_ptr (master->options, "acl", "on");
+                ret = dict_set_static_ptr (options, "acl", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key acl");
@@ -357,7 +330,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->selinux) {
-                ret = dict_set_static_ptr (master->options, "selinux", "on");
+                ret = dict_set_static_ptr (options, "selinux", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key selinux");
@@ -377,7 +350,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (cmd_args->read_only) {
-                ret = dict_set_static_ptr (master->options, "read-only", "on");
+                ret = dict_set_static_ptr (options, "read-only", "on");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key read-only");
@@ -386,7 +359,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
 	if (cmd_args->gid_timeout) {
-		ret = dict_set_int32(master->options, "gid-timeout",
+		ret = dict_set_int32(options, "gid-timeout",
 			cmd_args->gid_timeout);
 		if (ret < 0) {
 			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
@@ -394,10 +367,28 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
 			goto err;
 		}
 	}
+	if (cmd_args->background_qlen) {
+		ret = dict_set_int32 (options, "background-qlen",
+                                      cmd_args->background_qlen);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key background-qlen");
+			goto err;
+		}
+	}
+	if (cmd_args->congestion_threshold) {
+		ret = dict_set_int32 (options, "congestion-threshold",
+                                      cmd_args->congestion_threshold);
+		if (ret < 0) {
+			gf_log("glusterfsd", GF_LOG_ERROR, "failed to set dict "
+                               "value for key congestion-threshold");
+			goto err;
+		}
+	}
 
         switch (cmd_args->fuse_direct_io_mode) {
         case GF_OPTION_DISABLE: /* disable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "disable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -407,7 +398,7 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
                 }
                 break;
         case GF_OPTION_ENABLE: /* enable */
-                ret = dict_set_static_ptr (master->options, ZR_DIRECT_IO_OPT,
+                ret = dict_set_static_ptr (options, ZR_DIRECT_IO_OPT,
                                            "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
@@ -424,14 +415,64 @@ create_fuse_mount (glusterfs_ctx_t *ctx)
         }
 
         if (!cmd_args->no_daemon_mode) {
-                 ret = dict_set_static_ptr (master->options, "sync-to-mount",
-                                            "enable");
+                ret = dict_set_static_ptr (options, "sync-to-mount",
+                                           "enable");
                 if (ret < 0) {
                         gf_log ("glusterfsd", GF_LOG_ERROR,
                                 "failed to set dict value for key sync-mtab");
                         goto err;
                 }
         }
+        ret = 0;
+err:
+        return ret;
+}
+
+int
+create_fuse_mount (glusterfs_ctx_t *ctx)
+{
+        int              ret = 0;
+        cmd_args_t      *cmd_args = NULL;
+        xlator_t        *master = NULL;
+
+        cmd_args = &ctx->cmd_args;
+
+        if (!cmd_args->mount_point) {
+                gf_log ("", GF_LOG_TRACE,
+                        "mount point not found, not a client process");
+                return 0;
+        }
+
+        if (ctx->process_mode != GF_CLIENT_PROCESS) {
+                gf_log("glusterfsd", GF_LOG_ERROR,
+                       "Not a client process, not performing mount operation");
+                return -1;
+        }
+
+        master = GF_CALLOC (1, sizeof (*master),
+                            gfd_mt_xlator_t);
+        if (!master)
+                goto err;
+
+        master->name = gf_strdup ("fuse");
+        if (!master->name)
+                goto err;
+
+        if (xlator_set_type (master, "mount/fuse") == -1) {
+                gf_log ("glusterfsd", GF_LOG_ERROR,
+                        "MOUNT-POINT %s initialization failed",
+                        cmd_args->mount_point);
+                goto err;
+        }
+
+        master->ctx      = ctx;
+        master->options  = get_new_dict ();
+        if (!master->options)
+                goto err;
+
+        ret = set_fuse_mount_options (ctx, master->options);
+        if (ret)
+                goto err;
 
         ret = xlator_init (master);
         if (ret) {
@@ -848,6 +889,20 @@ parse_opts (int key, char *arg, struct argp_state *state)
 
 		argp_failure(state, -1, 0, "unknown group list timeout %s", arg);
 		break;
+        case ARGP_FUSE_BACKGROUND_QLEN_KEY:
+                if (!gf_string2int (arg, &cmd_args->background_qlen))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown background qlen option %s", arg);
+                break;
+        case ARGP_FUSE_CONGESTION_THRESHOLD_KEY:
+                if (!gf_string2int (arg, &cmd_args->congestion_threshold))
+                        break;
+
+                argp_failure (state, -1, 0,
+                              "unknown congestion threshold option %s", arg);
+                break;
 	}
 
         return 0;
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 03585cf22ee..b742df2d9f9 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -87,9 +87,10 @@ enum argp_option_keys {
         ARGP_USER_MAP_ROOT_KEY            = 156,
         ARGP_MEM_ACCOUNTING_KEY           = 157,
         ARGP_SELINUX_KEY                  = 158,
-        /* glusterfs upstream includes options 159-162, but we dont */
         ARGP_INODE32_KEY                  = 159, /* 163 upstream */
 	ARGP_GID_TIMEOUT_KEY		  = 160,
+	ARGP_FUSE_BACKGROUND_QLEN_KEY     = 161,
+	ARGP_FUSE_CONGESTION_THRESHOLD_KEY = 162,
 };
 
 struct _gfd_vol_top_priv_t {
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 153af2410a5..d331a5ee640 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -314,7 +314,8 @@ struct _cmd_args {
         pid_t            client_pid;
         int              client_pid_set;
         unsigned         uid_map_root;
-
+        int              background_qlen;
+        int              congestion_threshold;
 
 	/* key args */
 	char            *mount_point;
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 07859c60a4a..8b93a4d1fb0 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -3389,8 +3389,8 @@ fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
         if (fini->minor >= 13) {
                 /* these values seemed to work fine during testing */
-                fino.max_background = 64;
-                fino.congestion_threshold = 48;
+                fino.max_background = priv->background_qlen;
+                fino.congestion_threshold = priv->congestion_threshold;
         }
         if (fini->minor < 9)
                 *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
@@ -4574,6 +4574,32 @@ init (xlator_t *this_xl)
 		goto cleanup_exit;
 	}
 
+        /* default values seemed to work fine during testing */
+	GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
+                        cleanup_exit);
+	GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
+                        int32, cleanup_exit);
+
+        if (dict_get (this_xl->options, "background-qlen") &&
+            !dict_get (this_xl->options, "congestion-threshold")) {
+                /* user has set only background-qlen, not congestion-threshold,
+                   use the fuse kernel driver formula to set congestion,
+                   ie, 75% */
+                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control as 75%% of "
+                        "background-queue length (ie, (.75 * %d) = %d",
+                        priv->background_qlen, priv->congestion_threshold);
+        }
+        /* congestion should not be higher than background queue length */
+        if (priv->congestion_threshold > priv->background_qlen) {
+                gf_log (this_xl->name, GF_LOG_INFO,
+                        "setting congestion control same as "
+                        "background-queue length (%d)",
+                        priv->background_qlen);
+                priv->congestion_threshold = priv->background_qlen;
+        }
+
         cmd_args = &this_xl->ctx->cmd_args;
         fsname = cmd_args->volfile;
         if (!fsname && cmd_args->volfile_server) {
@@ -4742,5 +4768,17 @@ struct volume_options options[] = {
 	  .type = GF_OPTION_TYPE_INT,
 	  .default_value = "0"
 	},
+        { .key  = {"background-qlen"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "64",
+          .min = 16,
+          .max = (64 * GF_UNIT_KB),
+        },
+        { .key  = {"congestion-threshold"},
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "48",
+          .min = 12,
+          .max = (64 * GF_UNIT_KB),
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index f9d897c3037..d5b57e068df 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -122,6 +122,10 @@ struct fuse_private {
 
         /* For communicating with separate mount thread. */
         int                  status_pipe[2];
+
+        /* for fuse queue length and congestion threshold */
+        int background_qlen;
+        int congestion_threshold;
 };
 typedef struct fuse_private fuse_private_t;
 
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index f3d3839e8fa..833013abc76 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -109,6 +109,14 @@ start_glusterfs ()
 	cmd_line=$(echo "$cmd_line --gid-timeout=$gid_timeout");
     fi
 
+    if [ -n "$bg_qlen" ]; then
+	cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
+    fi
+
+    if [ -n "$cong_threshold" ]; then
+	cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
+    fi
+
     if [ -z "$volfile_loc" ]; then
         if  [ -n "$server_ip" ]; then
             if [ -n "$server_port" ]; then
@@ -137,7 +145,6 @@ start_glusterfs ()
     err=0;
     $cmd_line;
 
-
     inode=$(stat -c %i $mount_point 2>/dev/null);
 
     # this is required if the stat returns error
@@ -284,6 +291,8 @@ main ()
                             "backupvolfile-server")
                                 backupvolfile_server=$value ;;
 			    "gid-timeout")	gid_timeout=$value ;;
+			    "background-qlen")	bg_qlen=$value ;;
+			    "congestion-threshold")	cong_threshold=$value ;;
                             *) echo "unknown option $key (ignored)" ;;
                         esac
                 esac