summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRaghavendra G <raghavendra@zresearch.com>2009-04-03 06:30:54 -0700
committerAnand V. Avati <avati@amp.gluster.com>2009-04-03 19:09:35 +0530
commit6eb419f1ca68f536a700e63c1521e428560a0d9d (patch)
tree571e5bb852d90b94ddab0bcc5a7deaf26630c2a0
parentd3d7245970189629937f6330a6d946f5fbb454af (diff)
server-protocol - reimplement connection cleanup to happen in 2 phases
- first phase, which happens when POLLERR is received on transport, releases all locks, flushes all open fds. - second phase, which happens when both the transports of connection destroyed, destroys the containers like lock table, fd table along with the connection. - the first phase, clears up any references to transport held by translators like posix-locks(in the form of blocked locks) paving way for the second phase. Signed-off-by: Anand V. Avati <avati@amp.gluster.com>
-rw-r--r--libglusterfs/src/fd.c62
-rw-r--r--libglusterfs/src/fd.h3
-rw-r--r--xlators/protocol/server/src/server-helpers.c205
-rw-r--r--xlators/protocol/server/src/server-helpers.h1
-rw-r--r--xlators/protocol/server/src/server-protocol.c19
-rw-r--r--xlators/protocol/server/src/server-protocol.h3
6 files changed, 261 insertions, 32 deletions
diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index 918b6c14d..deb3c8001 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -129,26 +129,68 @@ gf_fd_fdtable_alloc (void)
return fdtable;
}
+fd_t **
+__gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+ fd_t **fds = NULL;
+
+ if (count == NULL) {
+ goto out;
+ }
+
+ fds = fdtable->fds;
+ fdtable->fds = calloc (fdtable->max_fds, sizeof (fd_t *));
+ *count = fdtable->max_fds;
+
+out:
+ return fds;
+}
+
+fd_t **
+gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count)
+{
+ fd_t **fds = NULL;
+ if (fdtable) {
+ pthread_mutex_lock (&fdtable->lock);
+ {
+ fds = __gf_fd_fdtable_get_all_fds (fdtable, count);
+ }
+ pthread_mutex_unlock (&fdtable->lock);
+ }
+
+ return fds;
+}
+
void
gf_fd_fdtable_destroy (fdtable_t *fdtable)
{
-
- int32_t i = 0;
+ struct list_head list = {0, };
+ fd_t *fd = NULL;
+ fd_t **fds = NULL;
+ uint32_t fd_count = 0;
+ int32_t i = 0;
+
+ INIT_LIST_HEAD (&list);
if (fdtable) {
pthread_mutex_lock (&fdtable->lock);
{
- for (i=0; i < fdtable->max_fds; i++) {
- if (fdtable->fds[i]) {
- fd_t *fd = fdtable->fds[i];
-
- fd_unref (fd);
- }
- }
-
+ fds = __gf_fd_fdtable_get_all_fds (fdtable, &fd_count);
FREE (fdtable->fds);
}
pthread_mutex_unlock (&fdtable->lock);
+
+ if (fds != NULL) {
+ for (i = 0; i < fd_count; i++) {
+ fd = fds[i];
+ if (fd != NULL) {
+ fd_unref (fd);
+ }
+ }
+
+ FREE (fds);
+ }
+
pthread_mutex_destroy (&fdtable->lock);
FREE (fdtable);
}
diff --git a/libglusterfs/src/fd.h b/libglusterfs/src/fd.h
index 0382ccdc2..2916a4481 100644
--- a/libglusterfs/src/fd.h
+++ b/libglusterfs/src/fd.h
@@ -77,6 +77,9 @@ gf_fd_unused_get (fdtable_t *fdtable, fd_t *fdptr);
int32_t
gf_fd_unused_get2 (fdtable_t *fdtable, fd_t *fdptr, int32_t fd);
+fd_t **
+gf_fd_fdtable_get_all_fds (fdtable_t *fdtable, uint32_t *count);
+
void
gf_fd_fdtable_destroy (fdtable_t *fdtable);
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 1fa400d5f..9a6932f67 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -398,19 +398,192 @@ out:
}
+static int32_t
+server_connection_cleanup_flush_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno)
+{
+ fd_t *fd = NULL;
+ fd = frame->local;
+
+ fd_unref (fd);
+ frame->local = NULL;
+
+ STACK_DESTROY (frame->root);
+ return 0;
+}
+
+
int
-server_connection_destroy (xlator_t *this, server_connection_t *conn)
+server_connection_cleanup (xlator_t *this, server_connection_t *conn)
{
+ call_frame_t *frame = NULL, *tmp_frame = NULL;
+ xlator_t *bound_xl = NULL;
+ server_state_t *state = NULL;
+ struct list_head file_lockers;
+ struct list_head dir_lockers;
+ struct _lock_table *ltable = NULL;
+ struct _locker *locker = NULL, *tmp = NULL;
+ struct flock flock = {0,};
+ fd_t *fd = NULL;
+ uint32_t fd_count = 0;
+ fd_t **fds = NULL;
+ int32_t i = 0;
+
+ bound_xl = (xlator_t *) (conn->bound_xl);
+
+ if (bound_xl) {
+ /* trans will have ref_count = 1 after this call, but its
+ ok since this function is called in
+ GF_EVENT_TRANSPORT_CLEANUP */
+ frame = create_frame (this, this->ctx->pool);
- call_frame_t *frame = NULL, *tmp_frame = NULL;
- xlator_t *bound_xl = NULL;
- int32_t ret = -1;
- server_state_t *state = NULL;
- struct list_head file_lockers;
- struct list_head dir_lockers;
+ pthread_mutex_lock (&(conn->lock));
+ {
+ if (conn->ltable) {
+ ltable = conn->ltable;
+ conn->ltable = gf_lock_table_new ();
+ }
+ }
+ pthread_mutex_unlock (&conn->lock);
+
+ INIT_LIST_HEAD (&file_lockers);
+ INIT_LIST_HEAD (&dir_lockers);
+
+ LOCK (&ltable->lock);
+ {
+ list_splice_init (&ltable->file_lockers,
+ &file_lockers);
+
+ list_splice_init (&ltable->dir_lockers, &dir_lockers);
+ }
+ UNLOCK (&ltable->lock);
+ free (ltable);
+
+ flock.l_type = F_UNLCK;
+ flock.l_start = 0;
+ flock.l_len = 0;
+ list_for_each_entry_safe (locker,
+ tmp, &file_lockers, lockers) {
+ tmp_frame = copy_frame (frame);
+ /*
+ pid = 0 is a special case that tells posix-locks
+ to release all locks from this transport
+ */
+ tmp_frame->root->pid = 0;
+ tmp_frame->root->trans = conn;
+
+ if (locker->fd) {
+ STACK_WIND (tmp_frame, server_nop_cbk,
+ bound_xl,
+ bound_xl->fops->finodelk,
+ locker->volume,
+ locker->fd, F_SETLK, &flock);
+ fd_unref (locker->fd);
+ } else {
+ STACK_WIND (tmp_frame, server_nop_cbk,
+ bound_xl,
+ bound_xl->fops->inodelk,
+ locker->volume,
+ &(locker->loc), F_SETLK, &flock);
+ loc_wipe (&locker->loc);
+ }
+
+ free (locker->volume);
+
+ list_del_init (&locker->lockers);
+ free (locker);
+ }
+
+ tmp = NULL;
+ locker = NULL;
+ list_for_each_entry_safe (locker, tmp, &dir_lockers, lockers) {
+ tmp_frame = copy_frame (frame);
+
+ tmp_frame->root->pid = 0;
+ tmp_frame->root->trans = conn;
+
+ if (locker->fd) {
+ STACK_WIND (tmp_frame, server_nop_cbk,
+ bound_xl,
+ bound_xl->fops->fentrylk,
+ locker->volume,
+ locker->fd, NULL,
+ ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+ fd_unref (locker->fd);
+ } else {
+ STACK_WIND (tmp_frame, server_nop_cbk,
+ bound_xl,
+ bound_xl->fops->entrylk,
+ locker->volume,
+ &(locker->loc), NULL,
+ ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+ loc_wipe (&locker->loc);
+ }
+
+ free (locker->volume);
+
+ list_del_init (&locker->lockers);
+ free (locker);
+ }
+
+ pthread_mutex_lock (&conn->lock);
+ {
+ if (conn->fdtable) {
+ fds = gf_fd_fdtable_get_all_fds (conn->fdtable,
+ &fd_count);
+ }
+ }
+ pthread_mutex_unlock (&conn->lock);
+
+ if (fds != NULL) {
+ for (i = 0;i < fd_count; i++) {
+ fd = fds[i];
+
+ if (fd != NULL) {
+ tmp_frame = copy_frame (frame);
+ tmp_frame->local = fd;
+
+ tmp_frame->root->pid = 0;
+ tmp_frame->root->trans = conn;
+ STACK_WIND (tmp_frame,
+ server_connection_cleanup_flush_cbk,
+ bound_xl,
+ bound_xl->fops->flush,
+ fd);
+ }
+ }
+ FREE (fds);
+ }
+
+ state = CALL_STATE (frame);
+ if (state)
+ free (state);
+ STACK_DESTROY (frame->root);
+ }
+
+ return 0;
+}
+
+
+int
+server_connection_destroy (xlator_t *this, server_connection_t *conn)
+{
+ call_frame_t *frame = NULL, *tmp_frame = NULL;
+ xlator_t *bound_xl = NULL;
+ int32_t ret = -1;
+ server_state_t *state = NULL;
+ struct list_head file_lockers;
+ struct list_head dir_lockers;
struct _lock_table *ltable = NULL;
struct _locker *locker = NULL, *tmp = NULL;
struct flock flock = {0,};
+ fd_t *fd = NULL;
+ int32_t i = 0;
+ fd_t **fds = NULL;
+ uint32_t fd_count = 0;
bound_xl = (xlator_t *) (conn->bound_xl);
@@ -518,12 +691,30 @@ server_connection_destroy (xlator_t *this, server_connection_t *conn)
pthread_mutex_lock (&(conn->lock));
{
if (conn->fdtable) {
+ fds = gf_fd_fdtable_get_all_fds (conn->fdtable,
+ &fd_count);
gf_fd_fdtable_destroy (conn->fdtable);
conn->fdtable = NULL;
}
}
pthread_mutex_unlock (&conn->lock);
+ if (fds != NULL) {
+ for (i = 0; i < fd_count; i++) {
+ fd = fds[i];
+ if (fd != NULL) {
+ tmp_frame = copy_frame (frame);
+ tmp_frame->local = fd;
+
+ STACK_WIND (tmp_frame,
+ server_connection_cleanup_flush_cbk,
+ bound_xl,
+ bound_xl->fops->flush,
+ fd);
+ }
+ }
+ FREE (fds);
+ }
}
gf_log (this->name, GF_LOG_INFO, "destroyed connection of %s",
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index eb26de84f..f26091ea2 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -74,4 +74,5 @@ gf_del_locker (struct _lock_table *table, const char *volume,
int32_t
gf_direntry_to_bin (dir_entry_t *head,
char **bufferp);
+
#endif /* __SERVER_HELPERS_H__ */
diff --git a/xlators/protocol/server/src/server-protocol.c b/xlators/protocol/server/src/server-protocol.c
index 1153a5451..5eee27b92 100644
--- a/xlators/protocol/server/src/server-protocol.c
+++ b/xlators/protocol/server/src/server-protocol.c
@@ -4286,27 +4286,15 @@ server_release (call_frame_t *frame, xlator_t *bound_xl,
state = CALL_STATE(frame);
state->fd_no = ntoh64 (req->fd);
- state->fd = gf_fd_fdptr_get (conn->fdtable,
- state->fd_no);
-
- GF_VALIDATE_OR_GOTO(bound_xl->name, state->fd, fail);
gf_fd_put (conn->fdtable,
state->fd_no);
gf_log (bound_xl->name, GF_LOG_DEBUG,
- "%"PRId64": RELEASE \'fd=%"PRId64" (%"PRId64")\'",
- frame->root->unique, state->fd_no, state->fd->inode->ino);
+ "%"PRId64": RELEASE \'fd=%"PRId64"\'",
+ frame->root->unique, state->fd_no);
- STACK_WIND (frame,
- server_release_cbk,
- BOUND_XL(frame),
- BOUND_XL(frame)->fops->flush,
- state->fd);
- return 0;
-fail:
- server_release_cbk (frame, NULL, frame->this,
- -1, EINVAL);
+ server_release_cbk (frame, NULL, frame->this, 0, 0);
return 0;
}
@@ -8269,6 +8257,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
ret = -1;
transport_disconnect (trans);
+ server_connection_cleanup (this, trans->xl_private);
}
break;
diff --git a/xlators/protocol/server/src/server-protocol.h b/xlators/protocol/server/src/server-protocol.h
index f00b584fb..2e6923fb4 100644
--- a/xlators/protocol/server/src/server-protocol.h
+++ b/xlators/protocol/server/src/server-protocol.h
@@ -83,6 +83,9 @@ int
server_connection_destroy (xlator_t *this, server_connection_t *conn);
int
+server_connection_cleanup (xlator_t *this, server_connection_t *conn);
+
+int
server_nop_cbk (call_frame_t *frame, void *cookie,
xlator_t *this, int32_t op_ret, int32_t op_errno);