From 41b9616435cbdf671805856e487e373060c9455b Mon Sep 17 00:00:00 2001
From: Mohit Agrawal <moagrawa@redhat.com>
Date: Mon, 27 Jul 2020 18:08:00 +0530
Subject: posix: Implement a janitor thread to close fd

Problem: In the commit fb20713b380e1df8d7f9e9df96563be2f9144fd6 we use
         syntask to close fd but we have found the patch is reducing the
         performance

Solution: Use janitor thread to close fd's and save the pfd ctx into
          ctx janitor list and also save the posix_xlator into pfd object to
          avoid the race condition during cleanup in brick_mux environment

Change-Id: Ifb3d18a854b267333a3a9e39845bfefb83fbc092
Fixes: #1396
Signed-off-by: Mohit Agrawal <moagrawa@redhat.com>
---
 xlators/storage/posix/src/posix-helpers.c | 93 +++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

(limited to 'xlators/storage/posix/src/posix-helpers.c')

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 850d6177de6..5d15ccb66fd 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1592,6 +1592,99 @@ unlock:
     return;
 }
 
+static struct posix_fd *
+janitor_get_next_fd(glusterfs_ctx_t *ctx)
+{
+    struct posix_fd *pfd = NULL;
+
+    while (list_empty(&ctx->janitor_fds)) {
+        if (ctx->pxl_count == 0) {
+            return NULL;
+        }
+
+        pthread_cond_wait(&ctx->fd_cond, &ctx->fd_lock);
+    }
+
+    pfd = list_first_entry(&ctx->janitor_fds, struct posix_fd, list);
+    list_del_init(&pfd->list);
+
+    return pfd;
+}
+
+static void
+posix_close_pfd(xlator_t *xl, struct posix_fd *pfd)
+{
+    THIS = xl;
+
+    if (pfd->dir == NULL) {
+        gf_msg_trace(xl->name, 0, "janitor: closing file fd=%d", pfd->fd);
+        sys_close(pfd->fd);
+    } else {
+        gf_msg_debug(xl->name, 0, "janitor: closing dir fd=%p", pfd->dir);
+        sys_closedir(pfd->dir);
+    }
+
+    GF_FREE(pfd);
+}
+
+static void *
+posix_ctx_janitor_thread_proc(void *data)
+{
+    xlator_t *xl;
+    struct posix_fd *pfd;
+    glusterfs_ctx_t *ctx = NULL;
+    struct posix_private *priv_fd;
+
+    ctx = data;
+
+    pthread_mutex_lock(&ctx->fd_lock);
+
+    while ((pfd = janitor_get_next_fd(ctx)) != NULL) {
+        pthread_mutex_unlock(&ctx->fd_lock);
+
+        xl = pfd->xl;
+        posix_close_pfd(xl, pfd);
+
+        pthread_mutex_lock(&ctx->fd_lock);
+
+        priv_fd = xl->private;
+        priv_fd->rel_fdcount--;
+        if (!priv_fd->rel_fdcount)
+            pthread_cond_signal(&priv_fd->fd_cond);
+    }
+
+    pthread_mutex_unlock(&ctx->fd_lock);
+
+    return NULL;
+}
+
+int
+posix_spawn_ctx_janitor_thread(xlator_t *this)
+{
+    int ret = 0;
+    glusterfs_ctx_t *ctx = NULL;
+
+    ctx = this->ctx;
+
+    pthread_mutex_lock(&ctx->fd_lock);
+    {
+        if (ctx->pxl_count++ == 0) {
+            ret = gf_thread_create(&ctx->janitor, NULL,
+                                   posix_ctx_janitor_thread_proc, ctx,
+                                   "posixctxjan");
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED,
+                       "spawning janitor thread failed");
+                ctx->pxl_count--;
+            }
+        }
+    }
+    pthread_mutex_unlock(&ctx->fd_lock);
+
+    return ret;
+}
+
 static int
 is_fresh_file(int64_t sec, int64_t ns)
 {
-- 
cgit