From 2af8fca492f3e1152177641b5e6ab9ea59ec0acd Mon Sep 17 00:00:00 2001 From: Poornima G Date: Thu, 16 Aug 2018 16:36:08 +0530 Subject: posix: use synctask for janitor With brick mux, the number of threads increases as the number of bricks increases. As an initiative to reduce the number of threads in brick mux scenario, replacing janitor thread to use synctask infra. Now close() and closedir() handle by separate janitor thread which is linked with glusterfs_ctx. Updates #475 Change-Id: I0c4aaf728125ab7264442fde59f3d08542785f73 Signed-off-by: Poornima G --- xlators/storage/posix/src/Makefile.am | 2 +- xlators/storage/posix/src/posix-common.c | 19 ++- xlators/storage/posix/src/posix-entry-ops.c | 1 - xlators/storage/posix/src/posix-helpers.c | 197 +++++++++++++++++++------ xlators/storage/posix/src/posix-inode-fd-ops.c | 37 +++-- xlators/storage/posix/src/posix-messages.h | 2 +- xlators/storage/posix/src/posix.h | 14 +- 7 files changed, 193 insertions(+), 79 deletions(-) (limited to 'xlators/storage/posix/src') diff --git a/xlators/storage/posix/src/Makefile.am b/xlators/storage/posix/src/Makefile.am index d8af6221e4e..c080a229ff3 100644 --- a/xlators/storage/posix/src/Makefile.am +++ b/xlators/storage/posix/src/Makefile.am @@ -17,7 +17,7 @@ noinst_HEADERS = posix.h posix-mem-types.h posix-handle.h posix-aio.h \ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \ - -I$(top_srcdir)/rpc/rpc-lib/src + -I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) -I$(top_srcdir)/glusterfsd/src diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c index 111542ef3ae..a68a0dda938 100644 --- a/xlators/storage/posix/src/posix-common.c +++ b/xlators/storage/posix/src/posix-common.c @@ -60,6 +60,7 @@ #include #include "posix-gfid-path.h" #include +#include "timer-wheel.h" extern char *marker_xattrs[]; #define ALIGN_SIZE 4096 @@ -968,15 +969,12 @@ posix_init(xlator_t *this) if (_private->health_check_interval) posix_spawn_health_check_thread(this); - pthread_mutex_init(&_private->janitor_lock, NULL); - pthread_cond_init(&_private->janitor_cond, NULL); - INIT_LIST_HEAD(&_private->janitor_fds); - - posix_spawn_janitor_thread(this); + posix_janitor_timer_start(this); pthread_mutex_init(&_private->fsync_mutex, NULL); pthread_cond_init(&_private->fsync_cond, NULL); INIT_LIST_HEAD(&_private->fsyncs); + posix_spawn_ctx_janitor_thread(this); ret = gf_thread_create(&_private->fsyncer, NULL, posix_fsyncer, this, "posixfsy"); @@ -1072,6 +1070,7 @@ posix_fini(xlator_t *this) { struct posix_private *priv = this->private; gf_boolean_t health_check = _gf_false; + int ret = 0; if (!priv) return; @@ -1093,8 +1092,13 @@ posix_fini(xlator_t *this) priv->disk_space_check = 0; } if (priv->janitor) { - (void)gf_thread_cleanup_xint(priv->janitor); - priv->janitor = 0; + /*TODO: Make sure the synctask is also complete */ + ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_TIMER_DELETE_FAILED, + "Failed to delete janitor timer"); + } + priv->janitor = NULL; } if (priv->fsyncer) { (void)gf_thread_cleanup_xint(priv->fsyncer); @@ -1106,7 +1110,6 @@ posix_fini(xlator_t *this) GF_FREE(priv->base_path); LOCK_DESTROY(&priv->lock); - pthread_mutex_destroy(&priv->janitor_lock); pthread_mutex_destroy(&priv->fsync_mutex); GF_FREE(priv->hostname); GF_FREE(priv->trash_path); diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c index ffd36d2c714..4d705586741 100644 --- a/xlators/storage/posix/src/posix-entry-ops.c +++ b/xlators/storage/posix/src/posix-entry-ops.c @@ -1366,7 +1366,6 @@ posix_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, priv->trash_path, gfid_str); gf_msg_debug(this->name, 0, "Moving %s to %s", real_path, tmp_path); op_ret = sys_rename(real_path, tmp_path); - pthread_cond_signal(&priv->janitor_cond); } } else { op_ret = sys_rmdir(real_path); diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 54fc1dc1195..208e319b336 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -57,6 +57,8 @@ #include "posix-gfid-path.h" #include #include "glusterfsd.h" +#include "glusterfs/syncop.h" +#include "timer-wheel.h" #include char *marker_xattrs[] = {"trusted.glusterfs.quota.*", @@ -1409,81 +1411,181 @@ janitor_walker(const char *fpath, const struct stat *sb, int typeflag, return 0; /* 0 = FTW_CONTINUE */ } -static struct posix_fd * -janitor_get_next_fd(xlator_t *this) +void +__posix_janitor_timer_start(xlator_t *this); + +static int +posix_janitor_task_done(int ret, call_frame_t *frame, void *data) { + xlator_t *this = NULL; struct posix_private *priv = NULL; - struct posix_fd *pfd = NULL; - struct timespec timeout; + this = data; + priv = this->private; + + LOCK(&priv->lock); + { + __posix_janitor_timer_start(this); + } + UNLOCK(&priv->lock); + + return 0; +} + +static int +posix_janitor_task(void *data) +{ + xlator_t *this = NULL; + struct posix_private *priv = NULL; + xlator_t *old_this = NULL; + + time_t now; + + this = data; + priv = this->private; + /* We need THIS to be set for janitor_walker */ + old_this = THIS; + THIS = this; + + time(&now); + if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { + if (priv->disable_landfill_purge) { + gf_msg_debug(this->name, 0, + "Janitor would have " + "cleaned out %s, but purge" + "is disabled.", + priv->trash_path); + } else { + gf_msg_trace(this->name, 0, "janitor cleaning out %s", + priv->trash_path); + + nftw(priv->trash_path, janitor_walker, 32, FTW_DEPTH | FTW_PHYS); + } + priv->last_landfill_check = now; + } + + THIS = old_this; + + return 0; +} + +static void +posix_janitor_task_initator(struct gf_tw_timer_list *timer, void *data, + unsigned long calltime) +{ + xlator_t *this = NULL; + int ret = 0; + + this = data; + + ret = synctask_new(this->ctx->env, posix_janitor_task, + posix_janitor_task_done, NULL, this); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, + "spawning janitor " + "thread failed"); + } + + return; +} + +void +__posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; + + priv = this->private; + timer = priv->janitor; + + INIT_LIST_HEAD(&timer->entry); + timer->expires = priv->janitor_sleep_duration; + timer->function = posix_janitor_task_initator; + timer->data = this; + gf_tw_add_timer(this->ctx->tw->timer_wheel, timer); + + return; +} + +void +posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; priv = this->private; - pthread_mutex_lock(&priv->janitor_lock); + LOCK(&priv->lock); { - if (list_empty(&priv->janitor_fds)) { + if (!priv->janitor) { + timer = GF_CALLOC(1, sizeof(struct gf_tw_timer_list), + gf_common_mt_tw_timer_list); + if (!timer) { + goto unlock; + } + priv->janitor = timer; + __posix_janitor_timer_start(this); + } + } +unlock: + UNLOCK(&priv->lock); + + return; +} + +static struct posix_fd * +janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep) +{ + struct posix_fd *pfd = NULL; + + struct timespec timeout; + + pthread_mutex_lock(&ctx->janitor_lock); + { + if (list_empty(&ctx->janitor_fds)) { time(&timeout.tv_sec); - timeout.tv_sec += priv->janitor_sleep_duration; + timeout.tv_sec += janitor_sleep; timeout.tv_nsec = 0; - pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_lock, + pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock, &timeout); goto unlock; } - pfd = list_entry(priv->janitor_fds.next, struct posix_fd, list); + pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list); - list_del(priv->janitor_fds.next); + list_del(ctx->janitor_fds.next); } unlock: - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); return pfd; } static void * -posix_janitor_thread_proc(void *data) +posix_ctx_janitor_thread_proc(void *data) { xlator_t *this = NULL; - struct posix_private *priv = NULL; struct posix_fd *pfd; - - time_t now; + glusterfs_ctx_t *ctx = NULL; + struct posix_private *priv = NULL; + int32_t sleep_duration = 0; this = data; - priv = this->private; - + ctx = THIS->ctx; THIS = this; + priv = this->private; + sleep_duration = priv->janitor_sleep_duration; while (1) { - time(&now); - if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { - if (priv->disable_landfill_purge) { - gf_msg_debug(this->name, 0, - "Janitor would have " - "cleaned out %s, but purge" - "is disabled.", - priv->trash_path); - } else { - gf_msg_trace(this->name, 0, "janitor cleaning out %s", - priv->trash_path); - - nftw(priv->trash_path, janitor_walker, 32, - FTW_DEPTH | FTW_PHYS); - } - priv->last_landfill_check = now; - } - - pfd = janitor_get_next_fd(this); + pfd = janitor_get_next_fd(ctx, sleep_duration); if (pfd) { if (pfd->dir == NULL) { gf_msg_trace(this->name, 0, "janitor: closing file fd=%d", pfd->fd); sys_close(pfd->fd); } else { - gf_msg_debug(this->name, 0, - "janitor: closing" - " dir fd=%p", + gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); sys_closedir(pfd->dir); } @@ -1496,18 +1598,25 @@ posix_janitor_thread_proc(void *data) } void -posix_spawn_janitor_thread(xlator_t *this) +posix_spawn_ctx_janitor_thread(xlator_t *this) { struct posix_private *priv = NULL; int ret = 0; + glusterfs_ctx_t *ctx = NULL; priv = this->private; + ctx = THIS->ctx; LOCK(&priv->lock); { - if (!priv->janitor_present) { - ret = gf_thread_create(&priv->janitor, NULL, - posix_janitor_thread_proc, this, "posixjan"); + if (!ctx->janitor) { + pthread_mutex_init(&ctx->janitor_lock, NULL); + pthread_cond_init(&ctx->janitor_cond, NULL); + INIT_LIST_HEAD(&ctx->janitor_fds); + + ret = gf_thread_create(&ctx->janitor, NULL, + posix_ctx_janitor_thread_proc, this, + "posixctxjan"); if (ret < 0) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, @@ -1515,8 +1624,6 @@ posix_spawn_janitor_thread(xlator_t *this) "thread failed"); goto unlock; } - - priv->janitor_present = _gf_true; } } unlock: diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c index dafd1855ef9..d2324823592 100644 --- a/xlators/storage/posix/src/posix-inode-fd-ops.c +++ b/xlators/storage/posix/src/posix-inode-fd-ops.c @@ -1236,8 +1236,7 @@ posix_releasedir(xlator_t *this, fd_t *fd) struct posix_fd *pfd = NULL; uint64_t tmp_pfd = 0; int ret = 0; - - struct posix_private *priv = NULL; + glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); @@ -1255,18 +1254,21 @@ posix_releasedir(xlator_t *this, fd_t *fd) goto out; } - priv = this->private; - if (!priv) - goto out; + ctx = THIS->ctx; - pthread_mutex_lock(&priv->janitor_lock); + pthread_mutex_lock(&ctx->janitor_lock); { INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &priv->janitor_fds); - pthread_cond_signal(&priv->janitor_cond); + list_add_tail(&pfd->list, &ctx->janitor_fds); + pthread_cond_signal(&ctx->janitor_cond); } - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); + + /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); + sys_closedir(pfd->dir); + GF_FREE(pfd); + */ out: return 0; } @@ -2337,11 +2339,13 @@ posix_release(xlator_t *this, fd_t *fd) struct posix_fd *pfd = NULL; int ret = -1; uint64_t tmp_pfd = 0; + glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); priv = this->private; + ctx = THIS->ctx; ret = fd_ctx_del(fd, this, &tmp_pfd); if (ret < 0) { @@ -2349,22 +2353,23 @@ posix_release(xlator_t *this, fd_t *fd) "pfd is NULL from fd=%p", fd); goto out; } - pfd = (struct posix_fd *)(long)tmp_pfd; + pfd = (struct posix_fd *)(long)tmp_pfd; if (pfd->dir) { gf_msg(this->name, GF_LOG_WARNING, 0, P_MSG_DIR_NOT_NULL, "pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd); } - if (!priv) - goto out; - pthread_mutex_lock(&priv->janitor_lock); + pthread_mutex_lock(&ctx->janitor_lock); { INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &priv->janitor_fds); - pthread_cond_signal(&priv->janitor_cond); + list_add_tail(&pfd->list, &ctx->janitor_fds); + pthread_cond_signal(&ctx->janitor_cond); } - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); + + if (!priv) + goto out; LOCK(&priv->lock); { diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h index 928723db8f9..32292756bc4 100644 --- a/xlators/storage/posix/src/posix-messages.h +++ b/xlators/storage/posix/src/posix-messages.h @@ -68,6 +68,6 @@ GLFS_MSGID(POSIX, P_MSG_XATTR_FAILED, P_MSG_NULL_GFID, P_MSG_FCNTL_FAILED, P_MSG_FALLOCATE_FAILED, P_MSG_STOREMDATA_FAILED, P_MSG_FETCHMDATA_FAILED, P_MSG_GETMDATA_FAILED, P_MSG_SETMDATA_FAILED, P_MSG_FRESHFILE, P_MSG_MUTEX_FAILED, - P_MSG_COPY_FILE_RANGE_FAILED); + P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED); #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h index a1ec996f4b2..97c9666ab44 100644 --- a/xlators/storage/posix/src/posix.h +++ b/xlators/storage/posix/src/posix.h @@ -151,9 +151,6 @@ struct posix_private { time_t last_landfill_check; int32_t janitor_sleep_duration; - struct list_head janitor_fds; - pthread_cond_t janitor_cond; - pthread_mutex_t janitor_lock; int64_t read_value; /* Total read, from init */ int64_t write_value; /* Total write, from init */ @@ -179,9 +176,9 @@ struct posix_private { */ gf_boolean_t background_unlink; - /* janitor thread which cleans up /.trash (created by replicate) */ - pthread_t janitor; - gf_boolean_t janitor_present; + /* janitor task which cleans up /.trash (created by replicate) */ + struct gf_tw_timer_list *janitor; + char *trash_path; /* lock for brick dir */ DIR *mount_lock; @@ -344,7 +341,7 @@ int posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key, data_t *value, int flags, struct iatt *stbuf, fd_t *_fd); void -posix_spawn_janitor_thread(xlator_t *this); +posix_janitor_timer_start(xlator_t *this); int posix_acl_xattr_set(xlator_t *this, const char *path, dict_t *xattr_req); int @@ -664,4 +661,7 @@ posix_cs_maintenance(xlator_t *this, fd_t *fd, loc_t *loc, int *pfd, int posix_check_dev_file(xlator_t *this, inode_t *inode, char *fop, int *op_errno); +void +posix_spawn_ctx_janitor_thread(xlator_t *this); + #endif /* _POSIX_H */ -- cgit