mount/fuse: expose auto-invalidation as a mount option

Auto invalidation is necessary when same (meta)data is shared/access
across multiple mounts. However, if (meta)data is not shared, all
relevant I/O goes through the cache of single mount and hence is
coherent with (meta)data on bricks always. So, fuse-auto-invalidation
can be disabled for this case which gives a huge performance boost for
workloads that write data and then immediately read the data they just
wrote.

From glusterfs --help,

<snip>
      --auto-invalidation[=BOOL]   controls whether fuse-kernel can
                             auto-invalidate attribute, dentry and page-cache.
                             Disable this only if same files/directories are
                             not accessed across two different mounts
                             concurrently [default: "on"]
</snip>

Details on how disabling auto-invalidation helped to reduce pgbench
init times can be found at [1]. Time taken for pgbench init of scale
8000 was 8340s. That will be an improvement of 86% (59280s vs 8340s)
with auto-invalidations turned off along with other
optimizations. Just disabling auto-invalidation contributed 56%
improvement by reducing the total time taken by 33260s.

[1] https://www.spinics.net/lists/gluster-devel/msg25907.html

Change-Id: I0ed730dba9064bd9c576ad1800170a21e100e1ce
Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
updates: bz#1664934

10 files changed, 77 insertions, 10 deletions

diff --git a/doc/glusterfs.8 b/doc/glusterfs.8
index 0538c00e570..e36bd6fbcfe 100644
--- a/doc/glusterfs.8
+++ b/doc/glusterfs.8
@@ -139,6 +139,11 @@ Enable fuse in-kernel writeback cache.
 \fB\-\-negative\-timeout=SECONDS\fR
 Set negative timeout to SECONDS in fuse kernel module (the default is 0).
 .TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on].
+.TP
 \fB\-\-volfile-check\fR
 Enable strict volume file checking.
 
diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8
index aa6327c2c73..bc1de2a8c80 100644
--- a/doc/glusterfsd.8
+++ b/doc/glusterfsd.8
@@ -107,6 +107,11 @@ Enable/Disable direct-io mode in fuse module [default: enable]
 .TP
 \fB\-\-resolve-gids\fR
 Resolve all auxiliary groups in fuse translator (max 32 otherwise)
+.TP
+\fB\-\-auto\-invalidation=BOOL\fR
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
 
 .SS "Miscellaneous Options"
 .PP
diff --git a/doc/mount.glusterfs.8 b/doc/mount.glusterfs.8
index 902b0c1ee5c..d3c3533e284 100644
--- a/doc/mount.glusterfs.8
+++ b/doc/mount.glusterfs.8
@@ -146,6 +146,11 @@ Enable fuse in-kernel writeback cache [default: off]
 .TP
 \fBattr\-times\-granularity=\fRNS
 Declare supported granularity of file attribute [default: 0]
+.TP
+\fBauto\-invalidation=\fRBOOL
+controls whether fuse-kernel can auto-invalidate attribute, dentry and
+page-cache. Disable this only if same files/directories are not
+accessed across two different mounts concurrently [default: on]
 .PP
 .SH FILES
 .TP
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index df5b9ed0b31..c1b66ef0d32 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -270,6 +270,11 @@ static struct argp_option gf_options[] = {
     {"fuse-flush-handle-interrupt", ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY,
      "BOOL", OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
      "handle interrupt in fuse FLUSH handler"},
+    {"auto-invalidation", ARGP_FUSE_AUTO_INVAL_KEY, "BOOL", OPTION_ARG_OPTIONAL,
+     "controls whether fuse-kernel can auto-invalidate "
+     "attribute, dentry and page-cache. "
+     "Disable this only if same files/directories are not accessed across "
+     "two different mounts concurrently [default: \"on\"]"},
     {0, 0, 0, 0, "Miscellaneous Options:"},
     {
         0,
@@ -623,6 +628,15 @@ set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
             goto err;
         }
     }
+
+    ret = dict_set_uint32(options, "auto-invalidation",
+                          cmd_args->fuse_auto_inval);
+    if (ret < 0) {
+        gf_msg("glusterfsd", GF_LOG_ERROR, 0, glusterfsd_msg_4,
+               "failed to set dict value for key auto-invalidation");
+        goto err;
+    }
+
     switch (cmd_args->kernel_writeback_cache) {
         case GF_OPTION_ENABLE:
             ret = dict_set_static_ptr(options, "kernel-writeback-cache", "on");
@@ -1483,6 +1497,16 @@ parse_opts(int key, char *arg, struct argp_state *state)
                          "unknown fuse flush handle interrupt setting \"%s\"",
                          arg);
             break;
+        case ARGP_FUSE_AUTO_INVAL_KEY:
+            if (!arg)
+                arg = "yes";
+
+            if (gf_string2boolean(arg, &b) == 0) {
+                cmd_args->fuse_auto_inval = b;
+                break;
+            }
+
+            break;
     }
     return 0;
 }
diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h
index 86ac61c1a92..35cf6d88b7a 100644
--- a/glusterfsd/src/glusterfsd.h
+++ b/glusterfsd/src/glusterfsd.h
@@ -110,6 +110,7 @@ enum argp_option_keys {
     ARGP_PRINT_LIBEXECDIR_KEY = 188,
     ARGP_FUSE_FLUSH_HANDLE_INTERRUPT_KEY = 189,
     ARGP_FUSE_LRU_LIMIT_KEY = 190,
+    ARGP_FUSE_AUTO_INVAL_KEY = 191,
 };
 
 struct _gfd_vol_top_priv {
diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h
index b87bd4481cf..7c6af090fd8 100644
--- a/libglusterfs/src/glusterfs/glusterfs.h
+++ b/libglusterfs/src/glusterfs/glusterfs.h
@@ -574,6 +574,7 @@ struct _cmd_args {
     uint32_t attr_times_granularity;
 
     int fuse_flush_handle_interrupt;
+    int fuse_auto_inval;
 };
 typedef struct _cmd_args cmd_args_t;
 
diff --git a/tests/basic/mount-options.disabled b/tests/basic/mount-options.disabled
index 2373e4461ce..a04c8686276 100644
--- a/tests/basic/mount-options.disabled
+++ b/tests/basic/mount-options.disabled
@@ -127,6 +127,9 @@ EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-transport=ib-verbs
 EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
 
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --auto-invalidation=off
+EXPECT_WITHIN $UMOUNT_TIMEOUT "Y" force_umount $M0
+
 TEST ! glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volfile-server-port=socket
 
 TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --volume-name=$V0
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index ff5831234b1..c3945d7a13c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -5067,6 +5067,9 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
         /* If user did not explicitly set --fopen-keep-cache[=off],
            then check if kernel support FUSE_AUTO_INVAL_DATA and ...
         */
+
+        priv->fopen_keep_cache = 1;
+
 #if FUSE_KERNEL_MINOR_VERSION >= 20
         if (fini->flags & FUSE_AUTO_INVAL_DATA) {
             /* ... enable fopen_keep_cache mode if supported.
@@ -5075,25 +5078,26 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
                    "Detected "
                    "support for FUSE_AUTO_INVAL_DATA. Enabling "
                    "fopen_keep_cache automatically.");
-            fino.flags |= FUSE_AUTO_INVAL_DATA;
-            priv->fopen_keep_cache = 1;
+
+            if (priv->fuse_auto_inval)
+                fino.flags |= FUSE_AUTO_INVAL_DATA;
         } else
 #endif
         {
-
-            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
-                   "No support "
-                   "for FUSE_AUTO_INVAL_DATA. Disabling "
-                   "fopen_keep_cache.");
-            /* ... else disable. */
-            priv->fopen_keep_cache = 0;
+            if (priv->fuse_auto_inval) {
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                       "No support for FUSE_AUTO_INVAL_DATA. Disabling "
+                       "fopen_keep_cache.");
+                /* ... else disable. */
+                priv->fopen_keep_cache = 0;
+            }
         }
     } else if (priv->fopen_keep_cache == 1) {
         /* If user explicitly set --fopen-keep-cache[=on],
            then enable FUSE_AUTO_INVAL_DATA if possible.
         */
 #if FUSE_KERNEL_MINOR_VERSION >= 20
-        if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+        if (priv->fuse_auto_inval && (fini->flags & FUSE_AUTO_INVAL_DATA)) {
             gf_log("glusterfs-fuse", GF_LOG_DEBUG,
                    "fopen_keep_cache "
                    "is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
@@ -6507,6 +6511,8 @@ init(xlator_t *this_xl)
     GF_OPTION_INIT("reader-thread-count", priv->reader_thread_count, uint32,
                    cleanup_exit);
 
+    GF_OPTION_INIT("auto-invalidation", priv->fuse_auto_inval, bool,
+                   cleanup_exit);
     GF_OPTION_INIT(ZR_ENTRY_TIMEOUT_OPT, priv->entry_timeout, double,
                    cleanup_exit);
 
@@ -6944,6 +6950,15 @@ struct volume_options options[] = {
         .description = "makes glusterfs invalidate kernel inodes after "
                        "reaching this limit (0 means 'unlimited')",
     },
+    {
+        .key = {"auto-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .description = "controls whether fuse-kernel can auto-invalidate "
+                       "attribute, dentry and page-cache. Disable this only "
+                       "if same files/directories are not accessed across "
+                       "two different mounts concurrently",
+    },
     {.key = {NULL}},
 };
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index b892113eb79..697bd8848e1 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -187,6 +187,7 @@ struct fuse_private {
     pthread_mutex_t interrupt_mutex;
 
     gf_boolean_t flush_handle_interrupt;
+    gf_boolean_t fuse_auto_inval;
 
     /* LRU Limit, if not set, default is 128k for now */
     uint32_t lru_limit;
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index d09a7cd663e..868a57f7ede 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -229,6 +229,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --reader-thread-count=$reader_thread_count");
     fi
 
+    if [ -n "$auto-invalidation" ]; then
+        cmd_line=$(echo "$cmd_line --auto-invalidation=$fuse_auto_invalidation");
+    fi
+
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
@@ -529,6 +533,9 @@ with_options()
         "reader-thread-count")
             reader_thread_count=$value
             ;;
+        "auto-invalidation")
+            fuse_auto_invalidation=$value
+	    ;;
         "no-root-squash")
             if [ $value = "yes" ] ||
                 [ $value = "on" ] ||