performance/md-cache: introduce an option to control invalidation of inodes

Explicit invalidation by calling inode_invalidate is necessary when
same (meta)data is shared/access across multiple mounts. Without an
explicit inode_invalidate call, caches in the mount which didn't
witness writes wouldn't be aware of changes as writes wouldn't have
passed through them. However, if (meta)data is not shared, all
relevant I/O goes through the cache of single mount and hence is
coherent with (meta)data on bricks always. So, explicit inode
invalidation can be disabled for this case which gives a huge
performance boost for workloads that write data and then immediately
read the data they just wrote. Note that otherwise, local writes
(which pass through the cache) will change ctime and cause unnecessary
invalidations.

The name of the option that controls this behavior is
"performance.global-cache-invalidation". This option is global and it
purges caches both in glusterfs and kernel stack for native FUSE
mounts. For non-native FUSE mounts, it purges cache only from
glusterfs stack. This option is effective only when
performance.stat-prefetch is on.

Note that there is a similar option "performance.cache-invalidation",
but the scope of that option is limited to quick-read and md-cache.

Change-Id: I462bb4b65ff9aae1f6ba76f50b1f2f94fb10323b
Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
updates: bz#1664934

2 files changed, 50 insertions, 10 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 7340fa07fca..46eda9379ac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1795,6 +1795,12 @@ struct volopt_map_entry glusterd_volopt_map[] = {
      .op_version = GD_OP_VERSION_3_9_0,
      .flags = VOLOPT_FLAG_CLIENT_OPT},
 
+    {.key = "performance.global-cache-invalidation",
+     .voltype = "performance/md-cache",
+     .option = "global-cache-invalidation",
+     .op_version = GD_OP_VERSION_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
     /* Feature translators */
     {.key = "features.uss",
      .voltype = "features/snapview-server",
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index ff5c8c2f914..a2d280e3290 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -71,6 +71,8 @@ struct mdc_conf {
     gf_boolean_t cache_swift_metadata;
     gf_boolean_t cache_samba_metadata;
     gf_boolean_t mdc_invalidation;
+    gf_boolean_t global_invalidation;
+
     time_t last_child_down;
     gf_lock_t lock;
     struct mdc_statistics mdc_counter;
@@ -470,6 +472,7 @@ mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
     uint32_t rollover = 0;
     uint64_t gen = 0;
     gf_boolean_t update_xa_time = _gf_false;
+    struct mdc_conf *conf = this->private;
 
     mdc = mdc_inode_prep(this, inode);
     if (!mdc) {
@@ -533,18 +536,20 @@ mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
             (iatt->ia_mtime_nsec != mdc->md_mtime_nsec) ||
             (iatt->ia_ctime != mdc->md_ctime) ||
             (iatt->ia_ctime_nsec != mdc->md_ctime_nsec)) {
-            if (!prebuf || (prebuf->ia_ctime != mdc->md_ctime) ||
-                (prebuf->ia_ctime_nsec != mdc->md_ctime_nsec) ||
-                (prebuf->ia_mtime != mdc->md_mtime) ||
-                (prebuf->ia_mtime_nsec != mdc->md_mtime_nsec)) {
-                gf_msg_trace("md-cache", 0,
-                             "prebuf doesn't "
-                             "match the value we have cached,"
-                             " invalidate the inode(%s)",
-                             uuid_utoa(inode->gfid));
+            if (conf->global_invalidation &&
+                (!prebuf || (prebuf->ia_mtime != mdc->md_mtime) ||
+                 (prebuf->ia_mtime_nsec != mdc->md_mtime_nsec) ||
+                 (prebuf->ia_ctime != mdc->md_ctime) ||
+                 (prebuf->ia_ctime_nsec != mdc->md_ctime_nsec))) {
+                if (IA_ISREG(inode->ia_type)) {
+                    gf_msg("md-cache", GF_LOG_TRACE, 0,
+                           MD_CACHE_MSG_DISCARD_UPDATE,
+                           "prebuf doesn't match the value we have cached,"
+                           " invalidate the inode(%s)",
+                           uuid_utoa(inode->gfid));
 
-                if (IA_ISREG(inode->ia_type))
                     inode_invalidate(inode);
+                }
             } else {
                 update_xa_time = _gf_true;
             }
@@ -3551,6 +3556,9 @@ mdc_reconfigure(xlator_t *this, dict_t *options)
     GF_OPTION_RECONF("cache-invalidation", conf->mdc_invalidation, options,
                      bool, out);
 
+    GF_OPTION_RECONF("global-cache-invalidation", conf->global_invalidation,
+                     options, bool, out);
+
     GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
 
     GF_OPTION_RECONF("md-cache-statfs", conf->cache_statfs, options, bool, out);
@@ -3622,6 +3630,9 @@ mdc_init(xlator_t *this)
 
     GF_OPTION_INIT("cache-invalidation", conf->mdc_invalidation, bool, out);
 
+    GF_OPTION_INIT("global-cache-invalidation", conf->global_invalidation, bool,
+                   out);
+
     GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
 
     pthread_mutex_init(&conf->statfs_cache.lock, NULL);
@@ -3859,6 +3870,29 @@ struct volume_options mdc_options[] = {
                        " on receiving the cache-invalidation notifications",
     },
     {
+        .key = {"global-cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {GD_OP_VERSION_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description =
+            "When \"on\", purges all read caches in kernel and glusterfs stack "
+            "whenever a stat change is detected. Stat changes can be detected "
+            "while processing responses to file operations (fop) or through "
+            "upcall notifications. Since purging caches can be an expensive "
+            "operation, it's advised to have this option \"on\" only when a "
+            "file "
+            "can be accessed from multiple different Glusterfs mounts and "
+            "caches across these different mounts are required to be coherent. "
+            "If a file is not accessed across different mounts "
+            "(simple example is having only one mount for a volume), its "
+            "advised to keep "
+            "this option \"off\" as all file modifications go through caches "
+            "keeping them "
+            "coherent. This option overrides value of "
+            "performance.cache-invalidation.",
+    },
+    {
         .key = {"md-cache-statfs"},
         .type = GF_OPTION_TYPE_BOOL,
         .default_value = "off",