gfapi: Register/Unregister Upcall events' callback

Polling continuously for upcall events is not optimal.
Hence new APIs have been added to allow applications to
register and unregister upcall events it is interested in
along with callback function to be invoked in case of any
such upcalls sent by backend server.

@TODO: Make changes in upcall xlator so that events are
sent to only those clients which either registered callbacks
or started polling. Shall be addressed in separate patch.

Updates: #315

Change-Id: I40473fd5cf689172ff2d7bb2869756b7fd5bc761
Signed-off-by: Soumya Koduri <skoduri@redhat.com>

1 files changed, 136 insertions, 50 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index 8d1ee9de1d7..72fd6975217 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -4398,29 +4398,151 @@ invalid_fs:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dup, 3.4.0);
 
+static void
+glfs_enqueue_upcall_data (struct glfs *fs, struct gf_upcall *upcall_data)
+{
+        int ret = -1;
+        upcall_entry *u_list = NULL;
+
+        if (!fs || !upcall_data)
+                goto out;
+
+        u_list = GF_CALLOC (1, sizeof(*u_list),
+                            glfs_mt_upcall_entry_t);
+
+        if (!u_list) {
+                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                        "Upcall entry allocation failed.");
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&u_list->upcall_list);
+
+        gf_uuid_copy (u_list->upcall_data.gfid, upcall_data->gfid);
+        u_list->upcall_data.event_type = upcall_data->event_type;
+
+        switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+                ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
+                                                          upcall_data);
+                break;
+        default:
+                break;
+        }
+
+        if (ret) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_INVALID_ENTRY,
+                        "Upcall entry validation failed.");
+                goto out;
+        }
+
+        pthread_mutex_lock (&fs->upcall_list_mutex);
+        {
+                list_add_tail (&u_list->upcall_list,
+                               &fs->upcall_list);
+        }
+        pthread_mutex_unlock (&fs->upcall_list_mutex);
+
+        ret = 0;
+
+out:
+        if (ret && u_list) {
+                GF_FREE (u_list->upcall_data.data);
+                GF_FREE(u_list);
+        }
+}
+
+static void
+glfs_cbk_upcall_data (struct glfs *fs, struct gf_upcall *upcall_data)
+{
+        int ret = -1;
+        struct glfs_upcall *up_arg = NULL;
+
+        if (!fs || !upcall_data)
+                goto out;
+
+        if (!(fs->upcall_events & upcall_data->event_type)) {
+                /* ignore events which application hasn't registered*/
+                goto out;
+        }
+
+        up_arg = GLFS_CALLOC (1, sizeof (struct gf_upcall),
+                              glfs_release_upcall,
+                              glfs_mt_upcall_entry_t);
+        if (!up_arg) {
+                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
+                        "Upcall entry allocation failed.");
+                goto out;
+        }
+
+        switch (upcall_data->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+                ret = glfs_h_poll_cache_invalidation (fs, up_arg, upcall_data);
+                break;
+        default:
+                errno = EINVAL;
+        }
+
+        if (!ret && (up_arg->reason != GLFS_UPCALL_EVENT_NULL)) {
+                /* It could so happen that the file which got
+                 * upcall notification may have got deleted by
+                 * the same client. In such cases up_arg->reason
+                 * is set to GLFS_UPCALL_EVENT_NULL. No need to
+                 * send upcall then */
+                (fs->up_cbk) (up_arg, fs->up_data);
+        } else if (up_arg->reason == GLFS_UPCALL_EVENT_NULL) {
+                gf_msg (THIS->name, GF_LOG_DEBUG, errno,
+                        API_MSG_INVALID_ENTRY,
+                        "Upcall_EVENT_NULL received. Skipping it.");
+                goto out;
+        } else {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        API_MSG_INVALID_ENTRY,
+                        "Upcall entry validation failed.");
+                goto out;
+        }
+
+        /* application takes care of calling glfs_free on up_arg post
+         * their processing */
+        ret = 0;
+
+out:
+        if (ret && up_arg) {
+                GLFS_FREE (up_arg);
+        }
+
+        return;
+}
+
 /*
  * This routine is called in case of any notification received
  * from the server. All the upcall events are queued up in a list
  * to be read by the applications.
  *
- * XXX: Applications may register a cbk function for each 'fs'
- * which then needs to be called by this routine incase of any
- * event received. The cbk fn is responsible for notifying the
+ * In case if the application registers a cbk function, that shall
+ * be called by this routine incase of any event received.
+ * The cbk fn is responsible for notifying the
  * applications the way it desires for each event queued (for eg.,
  * can raise a signal or broadcast a cond variable etc.)
+ *
+ * Otherwise all the upcall events are queued up in a list
+ * to be read/polled by the applications.
  */
 void
 priv_glfs_process_upcall_event (struct glfs *fs, void *data)
 {
-        int                ret             = -1;
-        upcall_entry       *u_list         = NULL;
         glusterfs_ctx_t    *ctx            = NULL;
         struct gf_upcall   *upcall_data    = NULL;
 
+        DECLARE_OLD_THIS;
+
         gf_msg_debug (THIS->name, 0,
                       "Upcall gfapi callback is called");
 
-        if (!fs || !data)
+        __GLFS_ENTRY_VALIDATE_FS (fs, err);
+
+        if (!data)
                 goto out;
 
         /* Unlike in I/O path, "glfs_fini" would not have freed
@@ -4441,48 +4563,16 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
         }
         pthread_mutex_unlock (&fs->mutex);
 
-
         upcall_data = (struct gf_upcall *)data;
 
-        gf_msg_trace (THIS->name, 0, "Upcall gfapi gfid = %s"
-                      "ret = %d", (char *)(upcall_data->gfid), ret);
+        gf_msg_trace (THIS->name, 0, "Upcall gfapi gfid = %s" ,
+                      (char *)(upcall_data->gfid));
 
-        u_list = GF_CALLOC (1, sizeof(*u_list),
-                            glfs_mt_upcall_entry_t);
-
-        if (!u_list) {
-                gf_msg (THIS->name, GF_LOG_ERROR, ENOMEM, API_MSG_ALLOC_FAILED,
-                        "Upcall entry allocation failed.");
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&u_list->upcall_list);
-
-        gf_uuid_copy (u_list->upcall_data.gfid, upcall_data->gfid);
-        u_list->upcall_data.event_type = upcall_data->event_type;
-
-        switch (upcall_data->event_type) {
-        case GF_UPCALL_CACHE_INVALIDATION:
-                ret = glfs_get_upcall_cache_invalidation (&u_list->upcall_data,
-                                                          upcall_data);
-                break;
-        default:
-                goto out;
-        }
-
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        API_MSG_INVALID_ENTRY,
-                        "Upcall entry validation failed.");
-                goto out;
-        }
-
-        pthread_mutex_lock (&fs->upcall_list_mutex);
-        {
-                list_add_tail (&u_list->upcall_list,
-                               &fs->upcall_list);
+        if (fs->up_cbk) { /* upcall cbk registered */
+                (void) glfs_cbk_upcall_data (fs, upcall_data);
+        } else {
+                (void) glfs_enqueue_upcall_data (fs, upcall_data);
         }
-        pthread_mutex_unlock (&fs->upcall_list_mutex);
 
         pthread_mutex_lock (&fs->mutex);
         {
@@ -4490,15 +4580,11 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data)
         }
         pthread_mutex_unlock (&fs->mutex);
 
-        ret = 0;
 out:
-        if (ret && u_list) {
-                GF_FREE (u_list->upcall_data.data);
-                GF_FREE(u_list);
-        }
+        __GLFS_EXIT_FS;
+err:
         return;
 }
-
 GFAPI_SYMVER_PRIVATE_DEFAULT(glfs_process_upcall_event, 3.7.0);
 
 ssize_t