gfapi: redesign the public interface for upcall consumers

The glfs_callback_arg and glfs_callback_inode_arg were allocated by
gfapi, and expected to be free()'d by the application. However it is not
reasonable to expect that applications use the same memory allocator to
as the compiled libgfapi.so. For instance, it is possible that gfapi
uses glibc malloc/free, and an application like NFS-Ganesha the versions
from jemalloc. Mismatching of the malloc() and free() functions causes
segmentation faults at best.

In order to prevent problems like this in the future, the API for
applications that consume upcalls has been remodeled. Any of the
structures that gfapi allocates, should be free'd with glfs_free(). The
members of the structures can not be accessed directly anymore, each
has its own function to access now.

Correcting the naming of the functions, structures and constants is a
continuation of commit 2775dc64101ed37c8d9809bf9852dbf0746ee2b6. These
new improvements not only have correct prefixes for the functions and
structures, the naming also reflects more to the upcall framework and
does not use "callback" anymore.

Cherry picked from commit 4721188a154acd9a0a4c096d8d73e97f3bf1b2a9:
> Change-Id: I2b8bd5a0a82036d2abea1a217f5e5975a1d4fe93
> BUG: 1344714
> Signed-off-by: Niels de Vos <ndevos@redhat.com>
> Reviewed-on: http://review.gluster.org/14701
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
> Reviewed-by: soumya k <skoduri@redhat.com>
> Reviewed-by: jiffin tony Thottan <jthottan@redhat.com>

Once difference with the version of this change in other branches is
that leases are not included in glusterfs-3.7. Hence there is a little
change that drops the handling of GF_UPCALL_RECALL_LEASE.

In addition, this backport contains commit 2775dc6410:
> libgfapi/upcall : prepend "glfs_" to callback_arg, callback_inode_arg
> Reviewed-on: http://review.gluster.org/14702

Change-Id: I2b8bd5a0a82036d2abea1a217f5e5975a1d4fe93
BUG: 1347715
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/15602
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Kaushal M <kaushal@redhat.com>

5 files changed, 103 insertions, 80 deletions

diff --git a/tests/basic/gfapi/bug1283983.c b/tests/basic/gfapi/bug1283983.c
index 76db8d5ca09..19354eb7453 100644
--- a/tests/basic/gfapi/bug1283983.c
+++ b/tests/basic/gfapi/bug1283983.c
@@ -32,20 +32,18 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        char      *filename = "/a1";
-        char      *filename2 = "/a2";
-        struct     stat sb = {0, };
-        struct    callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        int       cnt = 1;
-        struct callback_inode_arg *in_arg = NULL;
-        struct glfs_object *root = NULL, *leaf = NULL;
-
-        cbk.reason = 0;
+        glfs_t                   *fs = NULL;
+        int                       ret = 0, i;
+        glfs_fd_t                *fd = NULL;
+        char                     *filename = "/a1";
+        char                     *filename2 = "/a2";
+        struct stat               sb = {0, };
+        struct glfs_upcall       *cbk = NULL;
+        char                     *logfile = NULL;
+        char                     *volname = NULL;
+        int                       cnt = 1;
+        struct glfs_upcall_inode *in_arg = NULL;
+        struct glfs_object       *root = NULL, *leaf = NULL;
 
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 3) {
@@ -105,11 +103,13 @@ main (int argc, char *argv[])
                 LOG_ERR ("glfs_h_poll_upcall", ret);
 
                 /* There should not be any upcalls sent */
-                if (cbk.reason != GFAPI_CBK_EVENT_NULL) {
+                if (glfs_upcall_get_reason(cbk) != GLFS_UPCALL_EVENT_NULL) {
                         fprintf (stderr, "Error: Upcall received(%d)\n",
-                                 cbk.reason);
+                                 glfs_upcall_get_reason(cbk));
                         exit (1);
                 }
+
+                glfs_free (cbk);
         }
 
         ret = glfs_fini(fs);
diff --git a/tests/basic/gfapi/bug1291259.c b/tests/basic/gfapi/bug1291259.c
index 35f39938cb3..2169ba8c240 100644
--- a/tests/basic/gfapi/bug1291259.c
+++ b/tests/basic/gfapi/bug1291259.c
@@ -34,24 +34,22 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        glfs_t    *fs2 = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        char      *filename = "/a1";
-        char      *filename2 = "/a2";
-        struct     stat sb = {0, };
-        struct    callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        int        cnt = 1;
-        int       upcall_received = 0;
-        struct callback_inode_arg *in_arg = NULL;
-        struct glfs_object *root = NULL, *leaf = NULL;
-        unsigned char   globjhdl[GFAPI_HANDLE_LENGTH];
-        unsigned char   globjhdl2[GFAPI_HANDLE_LENGTH];
-
-        cbk.reason = 0;
+        glfs_t                     *fs = NULL;
+        glfs_t                     *fs2 = NULL;
+        int                         ret = 0, i;
+        glfs_fd_t                  *fd = NULL;
+        char                       *filename = "/a1";
+        char                       *filename2 = "/a2";
+        struct                      stat sb = {0, };
+        char                       *logfile = NULL;
+        char                       *volname = NULL;
+        char                       *hostname = NULL;
+        int                         cnt = 1;
+        int                         upcall_received = 0;
+        struct glfs_upcall         *cbk = NULL;
+        struct glfs_object         *root = NULL, *leaf = NULL;
+        unsigned char               globjhdl[GFAPI_HANDLE_LENGTH];
+        unsigned char               globjhdl2[GFAPI_HANDLE_LENGTH];
 
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 3) {
@@ -78,6 +76,12 @@ main (int argc, char *argv[])
         ret = glfs_init (fs);
         LOG_ERR("glfs_init", ret);
 
+        /* This does not block, but enables caching of events. Real
+         * applications like NFS-Ganesha run this in a thread before activity
+         * on the fs (through this instance) happens. */
+        ret = glfs_h_poll_upcall(fs, &cbk);
+        LOG_ERR ("glfs_h_poll_upcall", ret);
+
         fs2 = glfs_new (volname);
         if (!fs) {
                 fprintf (stderr, "glfs_new: returned NULL\n");
@@ -117,21 +121,30 @@ main (int argc, char *argv[])
         }
         fprintf (stderr, "glfs_h_create leaf - %p\n", leaf);
 
-        while (cnt++ < 5) {
+        while (cnt++ < 5 && !upcall_received) {
+                enum glfs_upcall_reason     reason = 0;
+                struct glfs_upcall_inode   *in_arg = NULL;
+
                 ret = glfs_h_poll_upcall(fs, &cbk);
                 LOG_ERR ("glfs_h_poll_upcall", ret);
+                if (ret)
+                        goto retry;
+
+                reason = glfs_upcall_get_reason (cbk);
+                fprintf (stderr, "Upcall received(%d)\n", reason);
+
+                if (reason == GLFS_UPCALL_INODE_INVALIDATE) {
+                        struct glfs_object *object = NULL;
 
-                if (cbk.reason == GFAPI_INODE_INVALIDATE) {
-                        fprintf (stderr, "Upcall received(%d)\n",
-                                 cbk.reason);
-                        in_arg = (struct callback_inode_arg *)(cbk.event_arg);
+                        in_arg = glfs_upcall_get_event (cbk);
+                        object = glfs_upcall_inode_get_object (in_arg);
 
                         ret = glfs_h_extract_handle (root,
                                                      globjhdl+GLAPI_UUID_LENGTH,
                                                      GFAPI_HANDLE_LENGTH);
                         LOG_ERR("glfs_h_extract_handle", (ret != 16));
 
-                        ret = glfs_h_extract_handle (in_arg->object,
+                        ret = glfs_h_extract_handle (object,
                                                   globjhdl2+GLAPI_UUID_LENGTH,
                                                   GFAPI_HANDLE_LENGTH);
                         LOG_ERR("glfs_h_extract_handle", (ret != 16));
@@ -143,6 +156,15 @@ main (int argc, char *argv[])
                         }
                         upcall_received = 1;
                 }
+
+retry:
+                if (!upcall_received)
+                        sleep (1); /* glfs_h_poll_upcall() does not block */
+
+                if (!ret) {
+                        glfs_free (cbk);
+                        cbk = NULL;
+                }
         }
 
         if (!upcall_received) {
diff --git a/tests/basic/gfapi/libgfapi-fini-hang.sh b/tests/basic/gfapi/libgfapi-fini-hang.sh
index 56633288020..cf964d87e31 100755
--- a/tests/basic/gfapi/libgfapi-fini-hang.sh
+++ b/tests/basic/gfapi/libgfapi-fini-hang.sh
@@ -21,7 +21,7 @@ EXPECT 'Created' volinfo_field $V0 'Status';
 TEST $CLI volume start $V0;
 EXPECT 'Started' volinfo_field $V0 'Status';
 
-build_tester -lgfapi $(dirname $0)/libgfapi-fini-hang.c -o $M0/libgfapi-fini-hang
+TEST build_tester $(dirname $0)/libgfapi-fini-hang.c -o $M0/libgfapi-fini-hang -lgfapi
 TEST cd $M0
  ./libgfapi-fini-hang $V0 &
 lpid=$!
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
index 13cca69da89..4d355e62278 100644
--- a/tests/basic/gfapi/upcall-cache-invalidate.c
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -9,7 +9,6 @@
 #include <errno.h>
 #include <glusterfs/api/glfs.h>
 #include <glusterfs/api/glfs-handles.h>
-int gfapi = 1;
 
 #define LOG_ERR(func, ret) do { \
         if (ret != 0) {            \
@@ -24,26 +23,24 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        glfs_t    *fs2 = NULL;
-        glfs_t    *fs_tmp = NULL;
-        glfs_t    *fs_tmp2 = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        glfs_fd_t *fd_tmp = NULL;
-        glfs_fd_t *fd_tmp2 = NULL;
-        char       readbuf[32];
-        char      *filename = "file_tmp";
-        char      *writebuf = NULL;
-        char      *vol_id  = NULL;
-        unsigned int       cnt = 1;
-        struct    callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        struct callback_inode_arg *in_arg = NULL;
-
-        cbk.reason = 0;
+        glfs_t                   *fs = NULL;
+        glfs_t                   *fs2 = NULL;
+        glfs_t                   *fs_tmp = NULL;
+        glfs_t                   *fs_tmp2 = NULL;
+        int                       ret = 0, i;
+        glfs_fd_t                *fd = NULL;
+        glfs_fd_t                *fd2 = NULL;
+        glfs_fd_t                *fd_tmp = NULL;
+        glfs_fd_t                *fd_tmp2 = NULL;
+        char                      readbuf[32];
+        char                     *filename = "file_tmp";
+        char                     *writebuf = NULL;
+        char                     *vol_id  = NULL;
+        unsigned int              cnt = 1;
+        struct glfs_upcall       *cbk = NULL;
+        char                     *logfile = NULL;
+        char                     *volname = NULL;
+        char                     *hostname = NULL;
 
         if (argc != 3) {
                 fprintf (stderr, "Invalid argument\n");
@@ -73,7 +70,6 @@ main (int argc, char *argv[])
          * on the fs (through this instance) happens. */
         ret = glfs_h_poll_upcall(fs_tmp, &cbk);
         LOG_ERR ("glfs_h_poll_upcall", ret);
-        cbk.reason = 0;
 
         fs2 = glfs_new (volname);
         if (!fs2) {
@@ -140,6 +136,7 @@ main (int argc, char *argv[])
                 ret = glfs_lseek (fd_tmp2, 0, SEEK_SET);
                 LOG_ERR ("glfs_lseek", ret);
 
+                memset (readbuf, 0, sizeof(readbuf));
                 ret = glfs_pread (fd_tmp2, readbuf, 4, 0, 0);
 
                 if (ret <= 0) {
@@ -153,26 +150,37 @@ main (int argc, char *argv[])
                  * there are I/Os on that fd
                  */
                 if (cnt > 2) {
+                        struct glfs_upcall_inode *in_arg = NULL;
+                        enum glfs_upcall_reason   reason = 0;
+                        struct glfs_object       *object = NULL;
+                        uint64_t                  flags = 0;
+                        uint64_t                  expire = 0;
+
                         ret = glfs_h_poll_upcall(fs_tmp, &cbk);
                         LOG_ERR ("glfs_h_poll_upcall", ret);
-                        /* Expect 'GFAPI_INODE_INVALIDATE' upcall event. */
-                        if (cbk.reason == GFAPI_INODE_INVALIDATE) {
-                                in_arg = cbk.event_arg;
+
+                        reason = glfs_upcall_get_reason (cbk);
+
+                        /* Expect 'GLFS_INODE_INVALIDATE' upcall event. */
+                        if (reason == GLFS_UPCALL_INODE_INVALIDATE) {
+                                in_arg = glfs_upcall_get_event (cbk);
+
+                                object = glfs_upcall_inode_get_object (in_arg);
+                                flags = glfs_upcall_inode_get_flags (in_arg);
+                                expire = glfs_upcall_inode_get_expire (in_arg);
+
                                 fprintf (stderr, " upcall event type - %d,"
                                          " object(%p), flags(%d), "
                                          " expire_time_attr(%d)\n" ,
-                                         cbk.reason, in_arg->object,
-                                         in_arg->flags,
-                                         in_arg->expire_time_attr);
-                                ret = glfs_h_close (in_arg->object);
-                                LOG_ERR ("glfs_h_close", ret);
-                                free (in_arg);
+                                         reason, object, flags, expire);
                         } else {
                                 fprintf (stderr,
-                                         "Dint receive upcall notify event");
+                                         "Didnt receive upcall notify event");
                                 ret = -1;
                                 goto err;
                         }
+
+                        glfs_free (cbk);
                 }
 
                 sleep(5);
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.sh b/tests/basic/gfapi/upcall-cache-invalidate.sh
index f6f59bea752..114074df711 100755
--- a/tests/basic/gfapi/upcall-cache-invalidate.sh
+++ b/tests/basic/gfapi/upcall-cache-invalidate.sh
@@ -5,13 +5,6 @@
 
 cleanup;
 
-# Upcall feature is disable for now. A new xlator option
-# will be introduced to turn it on. Skipping this test
-# till then.
-
-SKIP_TESTS;
-exit 0
-
 TEST glusterd
 
 TEST $CLI volume create $V0 localhost:$B0/brick1;