diff options
| -rw-r--r-- | glusterfsd/src/glusterfsd.c | 8 | ||||
| -rw-r--r-- | libglusterfs/src/glusterfs/glusterfs.h | 6 | ||||
| -rw-r--r-- | xlators/features/leases/src/leases.c | 6 | ||||
| -rw-r--r-- | xlators/storage/posix/src/Makefile.am | 2 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix-common.c | 19 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix-entry-ops.c | 1 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix-helpers.c | 197 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix-inode-fd-ops.c | 37 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix-messages.h | 2 | ||||
| -rw-r--r-- | xlators/storage/posix/src/posix.h | 14 |
10 files changed, 205 insertions, 87 deletions
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index 0dea52b6af1..7753c629216 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -2837,11 +2837,9 @@ main(int argc, char *argv[]) } /* do this _after_ daemonize() */ - if (cmd->global_timer_wheel) { - if (!glusterfs_ctx_tw_get(ctx)) { - ret = -1; - goto out; - } + if (!glusterfs_ctx_tw_get(ctx)) { + ret = -1; + goto out; } ret = glusterfs_volumes_init(ctx); diff --git a/libglusterfs/src/glusterfs/glusterfs.h b/libglusterfs/src/glusterfs/glusterfs.h index 325241818c0..ae137c5d334 100644 --- a/libglusterfs/src/glusterfs/glusterfs.h +++ b/libglusterfs/src/glusterfs/glusterfs.h @@ -716,6 +716,12 @@ struct _glusterfs_ctx { } stats; struct list_head volfile_list; + + /* Add members to manage janitor threads for cleanup fd */ + struct list_head janitor_fds; + pthread_cond_t janitor_cond; + pthread_mutex_t janitor_lock; + pthread_t janitor; }; typedef struct _glusterfs_ctx glusterfs_ctx_t; diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c index de74312b0ba..56299632e1d 100644 --- a/xlators/features/leases/src/leases.c +++ b/xlators/features/leases/src/leases.c @@ -1019,11 +1019,11 @@ fini(xlator_t *this) priv->inited_recall_thr = _gf_false; } - GF_FREE(priv); - if (this->ctx->tw) { + if (priv->timer_wheel) { glusterfs_ctx_tw_put(this->ctx); - this->ctx->tw = NULL; } + + GF_FREE(priv); return; } diff --git a/xlators/storage/posix/src/Makefile.am b/xlators/storage/posix/src/Makefile.am index d8af6221e4e..c080a229ff3 100644 --- a/xlators/storage/posix/src/Makefile.am +++ b/xlators/storage/posix/src/Makefile.am @@ -17,7 +17,7 @@ noinst_HEADERS = posix.h posix-mem-types.h posix-handle.h posix-aio.h \ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \ - -I$(top_srcdir)/rpc/rpc-lib/src + -I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) -I$(top_srcdir)/glusterfsd/src diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c index 111542ef3ae..a68a0dda938 100644 --- a/xlators/storage/posix/src/posix-common.c +++ b/xlators/storage/posix/src/posix-common.c @@ -60,6 +60,7 @@ #include <glusterfs/events.h> #include "posix-gfid-path.h" #include <glusterfs/compat-uuid.h> +#include "timer-wheel.h" extern char *marker_xattrs[]; #define ALIGN_SIZE 4096 @@ -968,15 +969,12 @@ posix_init(xlator_t *this) if (_private->health_check_interval) posix_spawn_health_check_thread(this); - pthread_mutex_init(&_private->janitor_lock, NULL); - pthread_cond_init(&_private->janitor_cond, NULL); - INIT_LIST_HEAD(&_private->janitor_fds); - - posix_spawn_janitor_thread(this); + posix_janitor_timer_start(this); pthread_mutex_init(&_private->fsync_mutex, NULL); pthread_cond_init(&_private->fsync_cond, NULL); INIT_LIST_HEAD(&_private->fsyncs); + posix_spawn_ctx_janitor_thread(this); ret = gf_thread_create(&_private->fsyncer, NULL, posix_fsyncer, this, "posixfsy"); @@ -1072,6 +1070,7 @@ posix_fini(xlator_t *this) { struct posix_private *priv = this->private; gf_boolean_t health_check = _gf_false; + int ret = 0; if (!priv) return; @@ -1093,8 +1092,13 @@ posix_fini(xlator_t *this) priv->disk_space_check = 0; } if (priv->janitor) { - (void)gf_thread_cleanup_xint(priv->janitor); - priv->janitor = 0; + /*TODO: Make sure the synctask is also complete */ + ret = gf_tw_del_timer(this->ctx->tw->timer_wheel, priv->janitor); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_TIMER_DELETE_FAILED, + "Failed to delete janitor timer"); + } + priv->janitor = NULL; } if (priv->fsyncer) { (void)gf_thread_cleanup_xint(priv->fsyncer); @@ -1106,7 +1110,6 @@ posix_fini(xlator_t *this) GF_FREE(priv->base_path); LOCK_DESTROY(&priv->lock); - pthread_mutex_destroy(&priv->janitor_lock); pthread_mutex_destroy(&priv->fsync_mutex); GF_FREE(priv->hostname); GF_FREE(priv->trash_path); diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c index ffd36d2c714..4d705586741 100644 --- a/xlators/storage/posix/src/posix-entry-ops.c +++ b/xlators/storage/posix/src/posix-entry-ops.c @@ -1366,7 +1366,6 @@ posix_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, priv->trash_path, gfid_str); gf_msg_debug(this->name, 0, "Moving %s to %s", real_path, tmp_path); op_ret = sys_rename(real_path, tmp_path); - pthread_cond_signal(&priv->janitor_cond); } } else { op_ret = sys_rmdir(real_path); diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index 54fc1dc1195..208e319b336 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -57,6 +57,8 @@ #include "posix-gfid-path.h" #include <glusterfs/events.h> #include "glusterfsd.h" +#include "glusterfs/syncop.h" +#include "timer-wheel.h" #include <sys/types.h> char *marker_xattrs[] = {"trusted.glusterfs.quota.*", @@ -1409,81 +1411,181 @@ janitor_walker(const char *fpath, const struct stat *sb, int typeflag, return 0; /* 0 = FTW_CONTINUE */ } -static struct posix_fd * -janitor_get_next_fd(xlator_t *this) +void +__posix_janitor_timer_start(xlator_t *this); + +static int +posix_janitor_task_done(int ret, call_frame_t *frame, void *data) { + xlator_t *this = NULL; struct posix_private *priv = NULL; - struct posix_fd *pfd = NULL; - struct timespec timeout; + this = data; + priv = this->private; + + LOCK(&priv->lock); + { + __posix_janitor_timer_start(this); + } + UNLOCK(&priv->lock); + + return 0; +} + +static int +posix_janitor_task(void *data) +{ + xlator_t *this = NULL; + struct posix_private *priv = NULL; + xlator_t *old_this = NULL; + + time_t now; + + this = data; + priv = this->private; + /* We need THIS to be set for janitor_walker */ + old_this = THIS; + THIS = this; + + time(&now); + if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { + if (priv->disable_landfill_purge) { + gf_msg_debug(this->name, 0, + "Janitor would have " + "cleaned out %s, but purge" + "is disabled.", + priv->trash_path); + } else { + gf_msg_trace(this->name, 0, "janitor cleaning out %s", + priv->trash_path); + + nftw(priv->trash_path, janitor_walker, 32, FTW_DEPTH | FTW_PHYS); + } + priv->last_landfill_check = now; + } + + THIS = old_this; + + return 0; +} + +static void +posix_janitor_task_initator(struct gf_tw_timer_list *timer, void *data, + unsigned long calltime) +{ + xlator_t *this = NULL; + int ret = 0; + + this = data; + + ret = synctask_new(this->ctx->env, posix_janitor_task, + posix_janitor_task_done, NULL, this); + if (ret < 0) { + gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, + "spawning janitor " + "thread failed"); + } + + return; +} + +void +__posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; + + priv = this->private; + timer = priv->janitor; + + INIT_LIST_HEAD(&timer->entry); + timer->expires = priv->janitor_sleep_duration; + timer->function = posix_janitor_task_initator; + timer->data = this; + gf_tw_add_timer(this->ctx->tw->timer_wheel, timer); + + return; +} + +void +posix_janitor_timer_start(xlator_t *this) +{ + struct posix_private *priv = NULL; + struct gf_tw_timer_list *timer = NULL; priv = this->private; - pthread_mutex_lock(&priv->janitor_lock); + LOCK(&priv->lock); { - if (list_empty(&priv->janitor_fds)) { + if (!priv->janitor) { + timer = GF_CALLOC(1, sizeof(struct gf_tw_timer_list), + gf_common_mt_tw_timer_list); + if (!timer) { + goto unlock; + } + priv->janitor = timer; + __posix_janitor_timer_start(this); + } + } +unlock: + UNLOCK(&priv->lock); + + return; +} + +static struct posix_fd * +janitor_get_next_fd(glusterfs_ctx_t *ctx, int32_t janitor_sleep) +{ + struct posix_fd *pfd = NULL; + + struct timespec timeout; + + pthread_mutex_lock(&ctx->janitor_lock); + { + if (list_empty(&ctx->janitor_fds)) { time(&timeout.tv_sec); - timeout.tv_sec += priv->janitor_sleep_duration; + timeout.tv_sec += janitor_sleep; timeout.tv_nsec = 0; - pthread_cond_timedwait(&priv->janitor_cond, &priv->janitor_lock, + pthread_cond_timedwait(&ctx->janitor_cond, &ctx->janitor_lock, &timeout); goto unlock; } - pfd = list_entry(priv->janitor_fds.next, struct posix_fd, list); + pfd = list_entry(ctx->janitor_fds.next, struct posix_fd, list); - list_del(priv->janitor_fds.next); + list_del(ctx->janitor_fds.next); } unlock: - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); return pfd; } static void * -posix_janitor_thread_proc(void *data) +posix_ctx_janitor_thread_proc(void *data) { xlator_t *this = NULL; - struct posix_private *priv = NULL; struct posix_fd *pfd; - - time_t now; + glusterfs_ctx_t *ctx = NULL; + struct posix_private *priv = NULL; + int32_t sleep_duration = 0; this = data; - priv = this->private; - + ctx = THIS->ctx; THIS = this; + priv = this->private; + sleep_duration = priv->janitor_sleep_duration; while (1) { - time(&now); - if ((now - priv->last_landfill_check) > priv->janitor_sleep_duration) { - if (priv->disable_landfill_purge) { - gf_msg_debug(this->name, 0, - "Janitor would have " - "cleaned out %s, but purge" - "is disabled.", - priv->trash_path); - } else { - gf_msg_trace(this->name, 0, "janitor cleaning out %s", - priv->trash_path); - - nftw(priv->trash_path, janitor_walker, 32, - FTW_DEPTH | FTW_PHYS); - } - priv->last_landfill_check = now; - } - - pfd = janitor_get_next_fd(this); + pfd = janitor_get_next_fd(ctx, sleep_duration); if (pfd) { if (pfd->dir == NULL) { gf_msg_trace(this->name, 0, "janitor: closing file fd=%d", pfd->fd); sys_close(pfd->fd); } else { - gf_msg_debug(this->name, 0, - "janitor: closing" - " dir fd=%p", + gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); sys_closedir(pfd->dir); } @@ -1496,18 +1598,25 @@ posix_janitor_thread_proc(void *data) } void -posix_spawn_janitor_thread(xlator_t *this) +posix_spawn_ctx_janitor_thread(xlator_t *this) { struct posix_private *priv = NULL; int ret = 0; + glusterfs_ctx_t *ctx = NULL; priv = this->private; + ctx = THIS->ctx; LOCK(&priv->lock); { - if (!priv->janitor_present) { - ret = gf_thread_create(&priv->janitor, NULL, - posix_janitor_thread_proc, this, "posixjan"); + if (!ctx->janitor) { + pthread_mutex_init(&ctx->janitor_lock, NULL); + pthread_cond_init(&ctx->janitor_cond, NULL); + INIT_LIST_HEAD(&ctx->janitor_fds); + + ret = gf_thread_create(&ctx->janitor, NULL, + posix_ctx_janitor_thread_proc, this, + "posixctxjan"); if (ret < 0) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_THREAD_FAILED, @@ -1515,8 +1624,6 @@ posix_spawn_janitor_thread(xlator_t *this) "thread failed"); goto unlock; } - - priv->janitor_present = _gf_true; } } unlock: diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c index dafd1855ef9..d2324823592 100644 --- a/xlators/storage/posix/src/posix-inode-fd-ops.c +++ b/xlators/storage/posix/src/posix-inode-fd-ops.c @@ -1236,8 +1236,7 @@ posix_releasedir(xlator_t *this, fd_t *fd) struct posix_fd *pfd = NULL; uint64_t tmp_pfd = 0; int ret = 0; - - struct posix_private *priv = NULL; + glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); @@ -1255,18 +1254,21 @@ posix_releasedir(xlator_t *this, fd_t *fd) goto out; } - priv = this->private; - if (!priv) - goto out; + ctx = THIS->ctx; - pthread_mutex_lock(&priv->janitor_lock); + pthread_mutex_lock(&ctx->janitor_lock); { INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &priv->janitor_fds); - pthread_cond_signal(&priv->janitor_cond); + list_add_tail(&pfd->list, &ctx->janitor_fds); + pthread_cond_signal(&ctx->janitor_cond); } - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); + + /*gf_msg_debug(this->name, 0, "janitor: closing dir fd=%p", pfd->dir); + sys_closedir(pfd->dir); + GF_FREE(pfd); + */ out: return 0; } @@ -2337,11 +2339,13 @@ posix_release(xlator_t *this, fd_t *fd) struct posix_fd *pfd = NULL; int ret = -1; uint64_t tmp_pfd = 0; + glusterfs_ctx_t *ctx = NULL; VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); priv = this->private; + ctx = THIS->ctx; ret = fd_ctx_del(fd, this, &tmp_pfd); if (ret < 0) { @@ -2349,22 +2353,23 @@ posix_release(xlator_t *this, fd_t *fd) "pfd is NULL from fd=%p", fd); goto out; } - pfd = (struct posix_fd *)(long)tmp_pfd; + pfd = (struct posix_fd *)(long)tmp_pfd; if (pfd->dir) { gf_msg(this->name, GF_LOG_WARNING, 0, P_MSG_DIR_NOT_NULL, "pfd->dir is %p (not NULL) for file fd=%p", pfd->dir, fd); } - if (!priv) - goto out; - pthread_mutex_lock(&priv->janitor_lock); + pthread_mutex_lock(&ctx->janitor_lock); { INIT_LIST_HEAD(&pfd->list); - list_add_tail(&pfd->list, &priv->janitor_fds); - pthread_cond_signal(&priv->janitor_cond); + list_add_tail(&pfd->list, &ctx->janitor_fds); + pthread_cond_signal(&ctx->janitor_cond); } - pthread_mutex_unlock(&priv->janitor_lock); + pthread_mutex_unlock(&ctx->janitor_lock); + + if (!priv) + goto out; LOCK(&priv->lock); { diff --git a/xlators/storage/posix/src/posix-messages.h b/xlators/storage/posix/src/posix-messages.h index 928723db8f9..32292756bc4 100644 --- a/xlators/storage/posix/src/posix-messages.h +++ b/xlators/storage/posix/src/posix-messages.h @@ -68,6 +68,6 @@ GLFS_MSGID(POSIX, P_MSG_XATTR_FAILED, P_MSG_NULL_GFID, P_MSG_FCNTL_FAILED, P_MSG_FALLOCATE_FAILED, P_MSG_STOREMDATA_FAILED, P_MSG_FETCHMDATA_FAILED, P_MSG_GETMDATA_FAILED, P_MSG_SETMDATA_FAILED, P_MSG_FRESHFILE, P_MSG_MUTEX_FAILED, - P_MSG_COPY_FILE_RANGE_FAILED); + P_MSG_COPY_FILE_RANGE_FAILED, P_MSG_TIMER_DELETE_FAILED); #endif /* !_GLUSTERD_MESSAGES_H_ */ diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h index a1ec996f4b2..97c9666ab44 100644 --- a/xlators/storage/posix/src/posix.h +++ b/xlators/storage/posix/src/posix.h @@ -151,9 +151,6 @@ struct posix_private { time_t last_landfill_check; int32_t janitor_sleep_duration; - struct list_head janitor_fds; - pthread_cond_t janitor_cond; - pthread_mutex_t janitor_lock; int64_t read_value; /* Total read, from init */ int64_t write_value; /* Total write, from init */ @@ -179,9 +176,9 @@ struct posix_private { */ gf_boolean_t background_unlink; - /* janitor thread which cleans up /.trash (created by replicate) */ - pthread_t janitor; - gf_boolean_t janitor_present; + /* janitor task which cleans up /.trash (created by replicate) */ + struct gf_tw_timer_list *janitor; + char *trash_path; /* lock for brick dir */ DIR *mount_lock; @@ -344,7 +341,7 @@ int posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key, data_t *value, int flags, struct iatt *stbuf, fd_t *_fd); void -posix_spawn_janitor_thread(xlator_t *this); +posix_janitor_timer_start(xlator_t *this); int posix_acl_xattr_set(xlator_t *this, const char *path, dict_t *xattr_req); int @@ -664,4 +661,7 @@ posix_cs_maintenance(xlator_t *this, fd_t *fd, loc_t *loc, int *pfd, int posix_check_dev_file(xlator_t *this, inode_t *inode, char *fop, int *op_errno); +void +posix_spawn_ctx_janitor_thread(xlator_t *this); + #endif /* _POSIX_H */ |