From 2af8fca492f3e1152177641b5e6ab9ea59ec0acd Mon Sep 17 00:00:00 2001 From: Poornima G Date: Thu, 16 Aug 2018 16:36:08 +0530 Subject: posix: use synctask for janitor With brick mux, the number of threads increases as the number of bricks increases. As an initiative to reduce the number of threads in brick mux scenario, replacing janitor thread to use synctask infra. Now close() and closedir() handle by separate janitor thread which is linked with glusterfs_ctx. Updates #475 Change-Id: I0c4aaf728125ab7264442fde59f3d08542785f73 Signed-off-by: Poornima G --- xlators/storage/posix/src/posix-helpers.c | 197 +++++++++++++++++++++++------- 1 file changed, 152 insertions(+), 45 deletions(-) (limited to 'xlators/storage/posix/src/posix-helpers.c') diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 54fc1dc1195..208e319b336 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -57,6 +57,8 @@ #include "posix-gfid-path.h" #include #include "glusterfsd.h" +#include "glusterfs/syncop.h" +#include "timer-wheel.h" #include char *marker_xattrs[] = {"trusted.glusterfs.quota.*", @@ -1409,81 +1411,181 @@ janitor_walker(const char *fpath, const struct stat *sb, int typeflag, return 0; /* 0 = FTW_CONTINUE */ } -static struct posix_fd * -janitor_get_next_fd(xlator_t *this) +void +__posix_janitor_timer_start(xlator_t *this); + +static int +posix_janitor_task_done(int ret, call_frame_t *frame, void *data) { + xlator_t *this = NULL; struct posix_private *priv = NULL; - struct posix_fd *pfd = NULL; - struct timespec timeout; + this = data; + priv = this->private; + + LOCK(&priv->lock); + { + __posix_janitor_timer_start(this); + } + UNLOCK(&priv->lock); + + return 0; +} + +static int +posix_janitor_task(void *data) +{ + xlator_t *this = NULL; + struct posix_private *priv = NULL; + xlator_t *old_this = NULL; + + time_t now; + + this = data; + priv = this->private; + /* We need THIS to be set for janitor_walker */ + old_this = THIS; + THIS = this; + + time(&now); + if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { + if (priv->disable_landfill_purge) { + gf_msg_debug(this->name, 0, + "Janitor would have " + "cleaned out %s, but purge" + "is disabled.", + priv->trash_path); + } else { + gf_msg_trace(this->name, 0, "janitor cleaning out %s", + priv->trash_path); + + nftw(priv->trash_path, janitor_walker, 32, FTW_DEPTH | FTW_PHYS); + } + priv->last_landfill_check = now; + } + + THIS = old_this; + + return 0; +} + +static void +posix_janitor_task_initator(struct gf_tw_timer_list *timer, void *data, + unsigned long calltime) +{ + xlator_t *this = NULL; + int ret = 0; + + this = data; + + ret = synctask_new(this->ctx->env, posix_janitor_task, + posix_janitor_task_done, NULL, this); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, + "spawning janitor " + "thread failed"); + } + + return; +} + +void +__posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; + + priv = this->private; + timer = priv->janitor; + + INIT_LIST_HEAD(&timer->entry); + timer->expires = priv->janitor_sleep_duration; + timer->function = posix_janitor_task_initator; + timer->data = this; + gf_tw_add_timer(this->ctx->tw->timer_wheel, timer); + + return; +} + +void +posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; priv = this->private; - pthread_mutex_lock(&priv->janitor_lock); + LOCK(&priv->lock); { - if (list_empty(&priv->janitor_fds)) { + if (!priv->janitor) { + timer = GF_CALLOC(1, sizeof(struct gf_tw_timer_list), + gf_common_mt_tw_timer_list); + if (!timer) { + goto unlock; + } + priv->janitor = timer; + __posix_janitor_timer_start(this); + } + } +unlock: + UNLOCK(&priv->lock); + + return; +} + +static struct posix_fd * +janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep) +{ + struct posix_fd *pfd = NULL; + + struct timespec timeout; + + pthread_mutex_lock(&ctx->janitor_lock); + { + if (list_empty(&ctx->janitor_fds)) { time(&timeout.tv_sec); - timeout.tv_sec += priv->janitor_sleep_duration; + timeout.tv_sec += janitor_sleep; timeout.tv_nsec = 0; - pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_lock, + pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock, &timeout); goto unlock; } - pfd = list_entry(priv->janitor_fds.next, struct posix_fd, list); + pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list); - list_del(priv->janitor_fds.next); + list_del(ctx->janitor_fds.next); } unlock: - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); return pfd; } static void * -posix_janitor_thread_proc(void *data) +posix_ctx_janitor_thread_proc(void *data) { xlator_t *this = NULL; - struct posix_private *priv = NULL; struct posix_fd *pfd; - - time_t now; + glusterfs_ctx_t *ctx = NULL; + struct posix_private *priv = NULL; + int32_t sleep_duration = 0; this = data; - priv = this->private; - + ctx = THIS->ctx; THIS = this; + priv = this->private; + sleep_duration = priv->janitor_sleep_duration; while (1) { - time(&now); - if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { - if (priv->disable_landfill_purge) { - gf_msg_debug(this->name, 0, - "Janitor would have " - "cleaned out %s, but purge" - "is disabled.", - priv->trash_path); - } else { - gf_msg_trace(this->name, 0, "janitor cleaning out %s", - priv->trash_path); - - nftw(priv->trash_path, janitor_walker, 32, - FTW_DEPTH | FTW_PHYS); - } - priv->last_landfill_check = now; - } - - pfd = janitor_get_next_fd(this); + pfd = janitor_get_next_fd(ctx, sleep_duration); if (pfd) { if (pfd->dir == NULL) { gf_msg_trace(this->name, 0, "janitor: closing file fd=%d", pfd->fd); sys_close(pfd->fd); } else { - gf_msg_debug(this->name, 0, - "janitor: closing" - " dir fd=%p", + gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); sys_closedir(pfd->dir); } @@ -1496,18 +1598,25 @@ posix_janitor_thread_proc(void *data) } void -posix_spawn_janitor_thread(xlator_t *this) +posix_spawn_ctx_janitor_thread(xlator_t *this) { struct posix_private *priv = NULL; int ret = 0; + glusterfs_ctx_t *ctx = NULL; priv = this->private; + ctx = THIS->ctx; LOCK(&priv->lock); { - if (!priv->janitor_present) { - ret = gf_thread_create(&priv->janitor, NULL, - posix_janitor_thread_proc, this, "posixjan"); + if (!ctx->janitor) { + pthread_mutex_init(&ctx->janitor_lock, NULL); + pthread_cond_init(&ctx->janitor_cond, NULL); + INIT_LIST_HEAD(&ctx->janitor_fds); + + ret = gf_thread_create(&ctx->janitor, NULL, + posix_ctx_janitor_thread_proc, this, + "posixctxjan"); if (ret < 0) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, @@ -1515,8 +1624,6 @@ posix_spawn_janitor_thread(xlator_t *this) "thread failed"); goto unlock; } - - priv->janitor_present = _gf_true; } } unlock: -- cgit