gfapi: redesign the public interface for upcall consumers

The glfs_callback_arg and glfs_callback_inode_arg were allocated by
gfapi, and expected to be free()'d by the application. However it is not
reasonable to expect that applications use the same memory allocator to
as the compiled libgfapi.so. For instance, it is possible that gfapi
uses glibc malloc/free, and an application like NFS-Ganesha the versions
from jemalloc. Mismatching of the malloc() and free() functions causes
segmentation faults at best.

In order to prevent problems like this in the future, the API for
applications that consume upcalls has been remodeled. Any of the
structures that gfapi allocates, should be free'd with glfs_free(). The
members of the structures can not be accessed directly anymore, each
has its own function to access now.

Correcting the naming of the functions, structures and constants is a
continuation of commit 2775dc64101ed37c8d9809bf9852dbf0746ee2b6. These
new improvements not only have correct prefixes for the functions and
structures, the naming also reflects more to the upcall framework and
does not use "callback" anymore.

Change-Id: I2b8bd5a0a82036d2abea1a217f5e5975a1d4fe93
BUG: 1344714
Signed-off-by: Niels de Vos <ndevos@redhat.com>
Reviewed-on: http://review.gluster.org/14701
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: soumya k <skoduri@redhat.com>
Reviewed-by: jiffin tony Thottan <jthottan@redhat.com>

5 files changed, 96 insertions, 80 deletions

diff --git a/tests/basic/gfapi/bug1283983.c b/tests/basic/gfapi/bug1283983.c
index 6583097697b..3334b290d9e 100644
--- a/tests/basic/gfapi/bug1283983.c
+++ b/tests/basic/gfapi/bug1283983.c
@@ -31,20 +31,18 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        char      *filename = "/a1";
-        char      *filename2 = "/a2";
-        struct     stat sb = {0, };
-        struct    glfs_callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        int       cnt = 1;
-        struct glfs_callback_inode_arg *in_arg = NULL;
-        struct glfs_object *root = NULL, *leaf = NULL;
-
-        cbk.reason = 0;
+        glfs_t                   *fs = NULL;
+        int                       ret = 0, i;
+        glfs_fd_t                *fd = NULL;
+        char                     *filename = "/a1";
+        char                     *filename2 = "/a2";
+        struct stat               sb = {0, };
+        struct glfs_upcall       *cbk = NULL;
+        char                     *logfile = NULL;
+        char                     *volname = NULL;
+        int                       cnt = 1;
+        struct glfs_upcall_inode *in_arg = NULL;
+        struct glfs_object       *root = NULL, *leaf = NULL;
 
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 4) {
@@ -105,11 +103,13 @@ main (int argc, char *argv[])
                 LOG_ERR ("glfs_h_poll_upcall", ret);
 
                 /* There should not be any upcalls sent */
-                if (cbk.reason != GFAPI_CBK_EVENT_NULL) {
+                if (glfs_upcall_get_reason(cbk) != GLFS_UPCALL_EVENT_NULL) {
                         fprintf (stderr, "Error: Upcall received(%d)\n",
-                                 cbk.reason);
+                                 glfs_upcall_get_reason(cbk));
                         exit (1);
                 }
+
+                glfs_free (cbk);
         }
 
         ret = glfs_fini(fs);
diff --git a/tests/basic/gfapi/bug1291259.c b/tests/basic/gfapi/bug1291259.c
index 9fbe02ba34d..78b95c6dcfd 100644
--- a/tests/basic/gfapi/bug1291259.c
+++ b/tests/basic/gfapi/bug1291259.c
@@ -33,23 +33,22 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        glfs_t    *fs2 = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        char      *filename = "/a1";
-        char      *filename2 = "/a2";
-        struct     stat sb = {0, };
-        struct    glfs_callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        char      *hostname = NULL;
-        int        cnt = 1;
-        int       upcall_received = 0;
-        struct glfs_callback_inode_arg *in_arg = NULL;
-        struct glfs_object *root = NULL, *leaf = NULL;
-        unsigned char   globjhdl[GFAPI_HANDLE_LENGTH];
-        unsigned char   globjhdl2[GFAPI_HANDLE_LENGTH];
+        glfs_t                     *fs = NULL;
+        glfs_t                     *fs2 = NULL;
+        int                         ret = 0, i;
+        glfs_fd_t                  *fd = NULL;
+        char                       *filename = "/a1";
+        char                       *filename2 = "/a2";
+        struct                      stat sb = {0, };
+        char                       *logfile = NULL;
+        char                       *volname = NULL;
+        char                       *hostname = NULL;
+        int                         cnt = 1;
+        int                         upcall_received = 0;
+        struct glfs_upcall         *cbk = NULL;
+        struct glfs_object         *root = NULL, *leaf = NULL;
+        unsigned char               globjhdl[GFAPI_HANDLE_LENGTH];
+        unsigned char               globjhdl2[GFAPI_HANDLE_LENGTH];
 
         fprintf (stderr, "Starting libgfapi_fini\n");
         if (argc != 4) {
@@ -82,7 +81,6 @@ main (int argc, char *argv[])
          * on the fs (through this instance) happens. */
         ret = glfs_h_poll_upcall(fs, &cbk);
         LOG_ERR ("glfs_h_poll_upcall", ret);
-        cbk.reason = 0;
 
         fs2 = glfs_new (volname);
         if (!fs) {
@@ -123,21 +121,30 @@ main (int argc, char *argv[])
         }
         fprintf (stderr, "glfs_h_create leaf - %p\n", leaf);
 
-        while (cnt++ < 5) {
+        while (cnt++ < 5 && !upcall_received) {
+                enum glfs_upcall_reason     reason = 0;
+                struct glfs_upcall_inode   *in_arg = NULL;
+
                 ret = glfs_h_poll_upcall(fs, &cbk);
                 LOG_ERR ("glfs_h_poll_upcall", ret);
+                if (ret)
+                        goto retry;
+
+                reason = glfs_upcall_get_reason (cbk);
+                fprintf (stderr, "Upcall received(%d)\n", reason);
+
+                if (reason == GLFS_UPCALL_INODE_INVALIDATE) {
+                        struct glfs_object *object = NULL;
 
-                if (cbk.reason == GFAPI_INODE_INVALIDATE) {
-                        fprintf (stderr, "Upcall received(%d)\n",
-                                 cbk.reason);
-                        in_arg = (struct glfs_callback_inode_arg *)(cbk.event_arg);
+                        in_arg = glfs_upcall_get_event (cbk);
+                        object = glfs_upcall_inode_get_object (in_arg);
 
                         ret = glfs_h_extract_handle (root,
                                                      globjhdl+GLAPI_UUID_LENGTH,
                                                      GFAPI_HANDLE_LENGTH);
                         LOG_ERR("glfs_h_extract_handle", (ret != 16));
 
-                        ret = glfs_h_extract_handle (in_arg->object,
+                        ret = glfs_h_extract_handle (object,
                                                   globjhdl2+GLAPI_UUID_LENGTH,
                                                   GFAPI_HANDLE_LENGTH);
                         LOG_ERR("glfs_h_extract_handle", (ret != 16));
@@ -149,6 +156,13 @@ main (int argc, char *argv[])
                         }
                         upcall_received = 1;
                 }
+
+retry:
+                if (!upcall_received)
+                        sleep (1); /* glfs_h_poll_upcall() does not block */
+
+                glfs_free (cbk);
+                cbk = NULL;
         }
 
         if (!upcall_received) {
diff --git a/tests/basic/gfapi/libgfapi-fini-hang.t b/tests/basic/gfapi/libgfapi-fini-hang.t
index 7c939872574..ba262a943ee 100755
--- a/tests/basic/gfapi/libgfapi-fini-hang.t
+++ b/tests/basic/gfapi/libgfapi-fini-hang.t
@@ -21,9 +21,9 @@ EXPECT 'Created' volinfo_field $V0 'Status';
 TEST $CLI volume start $V0;
 EXPECT 'Started' volinfo_field $V0 'Status';
 
-logdir=`gluster --print-logdir`
+logdir=$(gluster --print-logdir)
 
-TEST build_tester -lgfapi $(dirname $0)/libgfapi-fini-hang.c -o $M0/libgfapi-fini-hang
+TEST build_tester $(dirname $0)/libgfapi-fini-hang.c -o $M0/libgfapi-fini-hang -lgfapi
 TEST cd $M0
  ./libgfapi-fini-hang $H0 $V0 $logdir/libgfapi-fini-hang.log &
 PID=$!
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.c b/tests/basic/gfapi/upcall-cache-invalidate.c
index 44e186955e7..9add3381278 100644
--- a/tests/basic/gfapi/upcall-cache-invalidate.c
+++ b/tests/basic/gfapi/upcall-cache-invalidate.c
@@ -8,7 +8,6 @@
 #include <errno.h>
 #include <glusterfs/api/glfs.h>
 #include <glusterfs/api/glfs-handles.h>
-int gfapi = 1;
 
 #define LOG_ERR(func, ret) do { \
         if (ret != 0) {            \
@@ -23,25 +22,24 @@ int gfapi = 1;
 int
 main (int argc, char *argv[])
 {
-        glfs_t    *fs = NULL;
-        glfs_t    *fs2 = NULL;
-        glfs_t    *fs_tmp = NULL;
-        glfs_t    *fs_tmp2 = NULL;
-        int        ret = 0, i;
-        glfs_fd_t *fd = NULL;
-        glfs_fd_t *fd2 = NULL;
-        glfs_fd_t *fd_tmp = NULL;
-        glfs_fd_t *fd_tmp2 = NULL;
-        char       readbuf[32];
-        char      *filename = "file_tmp";
-        char      *writebuf = NULL;
-        char      *vol_id  = NULL;
-        unsigned int       cnt = 1;
-        struct    glfs_callback_arg cbk;
-        char      *logfile = NULL;
-        char      *volname = NULL;
-        char      *hostname = NULL;
-        struct glfs_callback_inode_arg *in_arg = NULL;
+        glfs_t                   *fs = NULL;
+        glfs_t                   *fs2 = NULL;
+        glfs_t                   *fs_tmp = NULL;
+        glfs_t                   *fs_tmp2 = NULL;
+        int                       ret = 0, i;
+        glfs_fd_t                *fd = NULL;
+        glfs_fd_t                *fd2 = NULL;
+        glfs_fd_t                *fd_tmp = NULL;
+        glfs_fd_t                *fd_tmp2 = NULL;
+        char                      readbuf[32];
+        char                     *filename = "file_tmp";
+        char                     *writebuf = NULL;
+        char                     *vol_id  = NULL;
+        unsigned int              cnt = 1;
+        struct glfs_upcall       *cbk = NULL;
+        char                     *logfile = NULL;
+        char                     *volname = NULL;
+        char                     *hostname = NULL;
 
         if (argc != 4) {
                 fprintf (stderr, "Invalid argument\n");
@@ -72,7 +70,6 @@ main (int argc, char *argv[])
          * on the fs (through this instance) happens. */
         ret = glfs_h_poll_upcall(fs_tmp, &cbk);
         LOG_ERR ("glfs_h_poll_upcall", ret);
-        cbk.reason = 0;
 
         fs2 = glfs_new (volname);
         if (!fs2) {
@@ -139,6 +136,7 @@ main (int argc, char *argv[])
                 ret = glfs_lseek (fd_tmp2, 0, SEEK_SET);
                 LOG_ERR ("glfs_lseek", ret);
 
+                memset (readbuf, 0, sizeof(readbuf));
                 ret = glfs_pread (fd_tmp2, readbuf, 4, 0, 0);
 
                 if (ret <= 0) {
@@ -152,26 +150,37 @@ main (int argc, char *argv[])
                  * there are I/Os on that fd
                  */
                 if (cnt > 2) {
+                        struct glfs_upcall_inode *in_arg = NULL;
+                        enum glfs_upcall_reason   reason = 0;
+                        struct glfs_object       *object = NULL;
+                        uint64_t                  flags = 0;
+                        uint64_t                  expire = 0;
+
                         ret = glfs_h_poll_upcall(fs_tmp, &cbk);
                         LOG_ERR ("glfs_h_poll_upcall", ret);
-                        /* Expect 'GFAPI_INODE_INVALIDATE' upcall event. */
-                        if (cbk.reason == GFAPI_INODE_INVALIDATE) {
-                                in_arg = cbk.event_arg;
+
+                        reason = glfs_upcall_get_reason (cbk);
+
+                        /* Expect 'GLFS_INODE_INVALIDATE' upcall event. */
+                        if (reason == GLFS_UPCALL_INODE_INVALIDATE) {
+                                in_arg = glfs_upcall_get_event (cbk);
+
+                                object = glfs_upcall_inode_get_object (in_arg);
+                                flags = glfs_upcall_inode_get_flags (in_arg);
+                                expire = glfs_upcall_inode_get_expire (in_arg);
+
                                 fprintf (stderr, " upcall event type - %d,"
                                          " object(%p), flags(%d), "
                                          " expire_time_attr(%d)\n" ,
-                                         cbk.reason, in_arg->object,
-                                         in_arg->flags,
-                                         in_arg->expire_time_attr);
-                                ret = glfs_h_close (in_arg->object);
-                                LOG_ERR ("glfs_h_close", ret);
-                                free (in_arg);
+                                         reason, object, flags, expire);
                         } else {
                                 fprintf (stderr,
-                                         "Dint receive upcall notify event");
+                                         "Didnt receive upcall notify event");
                                 ret = -1;
                                 goto err;
                         }
+
+                        glfs_free (cbk);
                 }
 
                 sleep(5);
diff --git a/tests/basic/gfapi/upcall-cache-invalidate.t b/tests/basic/gfapi/upcall-cache-invalidate.t
index 6acbab2c51a..5fd6a3332e7 100755
--- a/tests/basic/gfapi/upcall-cache-invalidate.t
+++ b/tests/basic/gfapi/upcall-cache-invalidate.t
@@ -5,13 +5,6 @@
 
 cleanup;
 
-# Upcall feature is disable for now. A new xlator option
-# will be introduced to turn it on. Skipping this test
-# till then.
-
-SKIP_TESTS;
-exit 0
-
 TEST glusterd
 
 TEST $CLI volume create $V0 $H0:$B0/brick1;