event_pool: Add the code to destroy the poller threads and event pool gracefully.

Change-Id: I49b6ceebb45773620c318fb5d20b81623db75ab6
BUG: 1093594
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: http://review.gluster.org/9691
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Shyamsundar Ranganathan <srangana@redhat.com>

1 files changed, 113 insertions, 0 deletions

diff --git a/libglusterfs/src/event.c b/libglusterfs/src/event.c
index 4dd0f991700..f19d43a0ab1 100644
--- a/libglusterfs/src/event.c
+++ b/libglusterfs/src/event.c
@@ -144,3 +144,116 @@ event_reconfigure_threads (struct event_pool *event_pool, int value)
 out:
         return ret;
 }
+
+int
+event_pool_destroy (struct event_pool *event_pool)
+{
+        int ret = -1;
+        int destroy = 0, activethreadcount = 0;
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                destroy = event_pool->destroy;
+                activethreadcount = event_pool->activethreadcount;
+        }
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        if (!destroy || (activethreadcount > 0))
+                goto out;
+
+        ret = event_pool->ops->event_pool_destroy (event_pool);
+out:
+        return ret;
+}
+
+int
+poller_destroy_handler (int fd, int idx, void *data,
+                       int poll_out, int poll_in, int poll_err)
+{
+        int readfd = -1;
+        char buf = '\0';
+
+        readfd = *(int *)data;
+        if (readfd < 0)
+                return -1;
+
+        while (read (readfd, &buf, 1) > 0) {
+        }
+        return 0;
+}
+
+/* This function destroys all the poller threads.
+ * Note: to be called before event_pool_destroy is called.
+ * The order in which cleaning is performed:
+ * - Register a pipe fd(this is for waking threads in poll()/epoll_wait())
+ * - Set the destroy mode, which this no new event registration will succede
+ * - Reconfigure the thread count to 0(this will succede only in destroy mode)
+ * - Wake up all the threads in poll() or epoll_wait(), so that they can
+ *   destroy themselves.
+ * - Wait for the thread to join(which will happen only after all the other
+ *   threads are destroyed)
+ */
+int
+event_dispatch_destroy (struct event_pool *event_pool)
+{
+        int  ret     = -1;
+        int  fd[2]   = {-1};
+        int  idx     = -1;
+        struct timespec   sleep_till = {0, };
+
+        GF_VALIDATE_OR_GOTO ("event", event_pool, out);
+
+        ret = pipe2 (fd, O_NONBLOCK);
+        if (ret < 0)
+                goto out;
+
+        /* From the main thread register an event on the pipe fd[0],
+         */
+        idx = event_register (event_pool, fd[0], poller_destroy_handler,
+                              &fd[1], 1, 0);
+        if (idx < 0)
+                goto out;
+
+        /* Enter the destroy mode first, set this before reconfiguring to 0
+         * threads, to prevent further reconfigure to thread count > 0.
+         */
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                event_pool->destroy = 1;
+        }
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        ret = event_reconfigure_threads (event_pool, 0);
+        if (ret < 0)
+                goto out;
+
+        /* Write something onto the write end of the pipe(fd[1]) so that
+         * poll wakes up and calls the handler, poller_destroy_handler()
+         */
+        pthread_mutex_lock (&event_pool->mutex);
+        {
+                /* Write to pipe(fd[1]) and then wait for 1 second or until
+                 * a poller thread that is dying, broadcasts.
+                 */
+                while (event_pool->activethreadcount > 0) {
+                        write (fd[1], "dummy", 6);
+                        sleep_till.tv_sec = time (NULL) + 1;
+                        ret = pthread_cond_timedwait (&event_pool->cond,
+                                                      &event_pool->mutex,
+                                                      &sleep_till);
+                }
+        }
+        pthread_mutex_unlock (&event_pool->mutex);
+
+        ret = event_unregister (event_pool, fd[0], idx);
+
+ out:
+        if (fd[0] != -1)
+                close (fd[0]);
+        if (fd[1] != -1)
+                close (fd[1]);
+
+        return ret;
+}