gfapi: do not cache upcalls if the application is not interested

When the volume option 'features.cache-invalidation' is enabled, upcall
events are sent from the brick process to the client. Even if the client
is not interested in upcall events itself, md-cache or other xlators may
benefit from them.

By adding a new 'cache_upcalls' boolean in the 'struct glfs', we can
enable the caching of upcalls when the application called
glfs_h_poll_upcall(). NFS-Ganesha sets up a thread for handling upcalls
in the initialization phase, and calls glfs_h_poll_upcall() before any
NFS-client accesses the NFS-export.

In the future there will be a more flexible registration API for
enabling certain kind of upcall events. Until that is available, this
should work just fine.

Verificatio of this change is not trivial within our current regression
test framework. The bug report contains a description on how to reliably
reproduce the problem with the glusterfs-coreutils.

Cherry picked from commit 218c9b033fa44eacbc27d87491abd830548b362e:
> Change-Id: I818595c92db50e6e48f7bfe287ee05103a4a30a2
> BUG: 1368842
> Signed-off-by: Niels de Vos <ndevos@redhat.com>
> Reviewed-on: http://review.gluster.org/15191
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Poornima G <pgurusid@redhat.com>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: soumya k <skoduri@redhat.com>
> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>

Change-Id: I818595c92db50e6e48f7bfe287ee05103a4a30a2
BUG: 1368841
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/15346
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: soumya k <skoduri@redhat.com>

5 files changed, 21 insertions, 3 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 18bac8195d0..9c31502d725 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -4325,7 +4325,8 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
         {
                 ctx = fs->ctx;
 
-                if (ctx->cleanup_started) {
+                /* if we're not interested in upcalls (anymore), skip them */
+                if (ctx->cleanup_started || !fs->cache_upcalls) {
                         pthread_mutex_unlock (&fs->mutex);
                         goto out;
                 }
diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c
index d09e81ddb1a..f9b4ee90267 100644
--- a/api/src/glfs-handleops.c
+++ b/api/src/glfs-handleops.c
@@ -2028,6 +2028,10 @@ pub_glfs_h_poll_upcall (struct glfs *fs, struct callback_arg *up_arg)
                 }
 
                 fs->pin_refcnt++;
+
+                /* once we call this function, the applications seems to be
+                 * interested in events, enable caching them */
+                fs->cache_upcalls = _gf_true;
         }
         pthread_mutex_unlock (&fs->mutex);
 
diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h
index 50f61cb5f71..19958d88ceb 100644
--- a/api/src/glfs-internal.h
+++ b/api/src/glfs-internal.h
@@ -210,6 +210,7 @@ struct glfs {
 
 	gf_boolean_t        migration_in_progress;
 
+        gf_boolean_t        cache_upcalls; /* add upcalls to the upcall_list? */
         struct list_head    upcall_list;
         pthread_mutex_t     upcall_list_mutex; /* mutex for upcall entry list */
 
diff --git a/tests/basic/gfapi/bug1291259.c b/tests/basic/gfapi/bug1291259.c
index 35f39938cb3..a0cd821a185 100644
--- a/tests/basic/gfapi/bug1291259.c
+++ b/tests/basic/gfapi/bug1291259.c
@@ -51,8 +51,6 @@ main (int argc, char *argv[])
         unsigned char   globjhdl[GFAPI_HANDLE_LENGTH];
         unsigned char   globjhdl2[GFAPI_HANDLE_LENGTH];
 
-        cbk.reason = 0;
-
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 3) {
                 fprintf (stderr, "Invalid argument\n");
@@ -78,6 +76,13 @@ main (int argc, char *argv[])
         ret = glfs_init (fs);
         LOG_ERR("glfs_init", ret);
 
+        /* This does not block, but enables caching of events. Real
+         * applications like NFS-Ganesha run this in a thread before activity
+         * on the fs (through this instance) happens. */
+        ret = glfs_h_poll_upcall(fs, &cbk);
+        LOG_ERR ("glfs_h_poll_upcall", ret);
+        cbk.reason = 0;
+
         fs2 = glfs_new (volname);
         if (!fs) {
                 fprintf (stderr, "glfs_new: returned NULL\n");
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
index cc2e6a0fe33..13cca69da89 100644
--- a/tests/basic/gfapi/upcall-cache-invalidate.c
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -68,6 +68,13 @@ main (int argc, char *argv[])
         ret = glfs_init (fs);
         LOG_ERR("glfs_init", ret);
 
+        /* This does not block, but enables caching of events. Real
+         * applications like NFS-Ganesha run this in a thread before activity
+         * on the fs (through this instance) happens. */
+        ret = glfs_h_poll_upcall(fs_tmp, &cbk);
+        LOG_ERR ("glfs_h_poll_upcall", ret);
+        cbk.reason = 0;
+
         fs2 = glfs_new (volname);
         if (!fs2) {
                 fprintf (stderr, "glfs_new fs2: returned NULL\n");