From 5979bc3b964adcb8a536dc354e4eca08acaff03e Mon Sep 17 00:00:00 2001 From: Amar Tumballi Date: Thu, 7 Oct 2010 05:56:22 +0000 Subject: protocol/rpc/transport: bring in one more event for 'TRANSPORT-DESTROY' needed because, a RPC disconnect doesn't mean that a RPC transport/listener is dead. With this, the race in server protocol cleaning up the lock table / fd table when some frames are in transit will be handled properly. Signed-off-by: Amar Tumballi Signed-off-by: Vijay Bellur BUG: 1843 () URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1843 --- glusterfsd/src/glusterfsd.c | 2 +- rpc/rpc-lib/src/rpc-clnt.c | 43 +++++++++++++++------------- rpc/rpc-lib/src/rpc-clnt.h | 2 -- rpc/rpc-lib/src/rpc-transport.c | 3 +- rpc/rpc-lib/src/rpc-transport.h | 3 -- rpc/rpc-lib/src/rpcsvc-common.h | 1 + rpc/rpc-lib/src/rpcsvc.c | 9 +++++- rpc/rpc-transport/socket/src/socket.c | 2 +- xlators/protocol/client/src/client.c | 2 +- xlators/protocol/server/src/server-helpers.c | 6 +++- xlators/protocol/server/src/server.c | 7 ++++- 11 files changed, 47 insertions(+), 33 deletions(-) diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c index eb2f61ba5..4d8eb22fe 100644 --- a/glusterfsd/src/glusterfsd.c +++ b/glusterfsd/src/glusterfsd.c @@ -661,7 +661,7 @@ cleanup_and_exit (int signum) glusterfs_mgmt_pmap_signout (ctx); if (ctx->mgmt) - rpc_clnt_destroy (ctx->mgmt); + rpc_clnt_unref (ctx->mgmt); gf_log ("glusterfsd", GF_LOG_NORMAL, "shutting down"); diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index 61bc5263d..bd878f4fc 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -1428,32 +1428,16 @@ rpc_clnt_ref (struct rpc_clnt *rpc) return rpc; } -struct rpc_clnt * -rpc_clnt_unref (struct rpc_clnt *rpc) -{ - int count = 0; - - if (!rpc) - return NULL; - pthread_mutex_lock (&rpc->lock); - { - count = --rpc->refcount; - } - pthread_mutex_unlock (&rpc->lock); - if (!count) { - rpc_clnt_destroy (rpc); - return NULL; - } - return rpc; -} -void +static void rpc_clnt_destroy (struct rpc_clnt *rpc) { if (!rpc) return; - rpc_transport_destroy (rpc->conn.trans); + if (rpc->conn.trans) + rpc_transport_unref (rpc->conn.trans); + rpc_clnt_connection_cleanup (&rpc->conn); rpc_clnt_reconnect_cleanup (&rpc->conn); saved_frames_destroy (rpc->conn.saved_frames); @@ -1469,6 +1453,25 @@ rpc_clnt_destroy (struct rpc_clnt *rpc) return; } +struct rpc_clnt * +rpc_clnt_unref (struct rpc_clnt *rpc) +{ + int count = 0; + + if (!rpc) + return NULL; + pthread_mutex_lock (&rpc->lock); + { + count = --rpc->refcount; + } + pthread_mutex_unlock (&rpc->lock); + if (!count) { + rpc_clnt_destroy (rpc); + return NULL; + } + return rpc; +} + void rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config) diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h index f5fe17f5a..932497b13 100644 --- a/rpc/rpc-lib/src/rpc-clnt.h +++ b/rpc/rpc-lib/src/rpc-clnt.h @@ -223,8 +223,6 @@ rpc_clnt_ref (struct rpc_clnt *rpc); struct rpc_clnt * rpc_clnt_unref (struct rpc_clnt *rpc); -void rpc_clnt_destroy (struct rpc_clnt *rpc); - void rpc_clnt_set_connected (rpc_clnt_connection_t *conn); void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn); diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c index c69237d58..43b2a0c18 100644 --- a/rpc/rpc-lib/src/rpc-transport.c +++ b/rpc/rpc-lib/src/rpc-transport.c @@ -1110,8 +1110,7 @@ rpc_transport_unref (rpc_transport_t *this) pthread_mutex_unlock (&this->lock); if (refcount == 0) { - /* xlator_notify (this->xl, GF_EVENT_RPC_TRANSPORT_CLEANUP, - this); */ + this->notify (this, this->mydata, RPC_TRANSPORT_CLEANUP, NULL); rpc_transport_destroy (this); } diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h index 2ba46fba9..478de9ef1 100644 --- a/rpc/rpc-lib/src/rpc-transport.h +++ b/rpc/rpc-lib/src/rpc-transport.h @@ -248,9 +248,6 @@ int32_t rpc_transport_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply); -int32_t -rpc_transport_destroy (rpc_transport_t *this); - rpc_transport_t * rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *name); diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h index 7e72bc3ae..442049433 100644 --- a/rpc/rpc-lib/src/rpcsvc-common.h +++ b/rpc/rpc-lib/src/rpcsvc-common.h @@ -29,6 +29,7 @@ typedef enum { RPCSVC_EVENT_ACCEPT, RPCSVC_EVENT_DISCONNECT, + RPCSVC_EVENT_TRANSPORT_DESTROY, RPCSVC_EVENT_LISTENER_DEAD, } rpcsvc_event_t; diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c index b78b1f0e2..e14313fe7 100644 --- a/rpc/rpc-lib/src/rpcsvc.c +++ b/rpc/rpc-lib/src/rpcsvc.c @@ -1066,6 +1066,7 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata, rpc_transport_pollin_t *msg = NULL; rpc_transport_t *new_trans = NULL; rpcsvc_t *svc = NULL; + rpcsvc_listener_t *listener = NULL; svc = mydata; if (svc == NULL) { @@ -1101,7 +1102,13 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata, break; case RPC_TRANSPORT_CLEANUP: - /* FIXME: think about this later */ + listener = rpcsvc_get_listener (svc, -1, trans); + if (listener == NULL) { + goto out; + } + + rpcsvc_program_notify (listener, RPCSVC_EVENT_TRANSPORT_DESTROY, + trans); ret = 0; break; diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c index ae4bd56a8..40531ad4c 100644 --- a/rpc/rpc-transport/socket/src/socket.c +++ b/rpc/rpc-transport/socket/src/socket.c @@ -1736,7 +1736,7 @@ socket_event_handler (int fd, int idx, void *data, if ((ret < 0) || poll_err) { gf_log ("transport", GF_LOG_TRACE, "disconnecting now"); socket_event_poll_err (this); - //rpc_transport_unref (this); + rpc_transport_unref (this); } out: diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c index 50cb917aa..1cc96532b 100644 --- a/xlators/protocol/client/src/client.c +++ b/xlators/protocol/client/src/client.c @@ -2008,7 +2008,7 @@ fini (xlator_t *this) if (conf) { if (conf->rpc) - rpc_clnt_destroy (conf->rpc); + rpc_clnt_unref (conf->rpc); /* Saved Fds */ /* TODO: */ diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c index 5ea6f5619..69fb5f6a7 100644 --- a/xlators/protocol/server/src/server-helpers.c +++ b/xlators/protocol/server/src/server-helpers.c @@ -102,6 +102,10 @@ free_state (server_state_t *state) state->conn = NULL; } + if (state->xprt) { + rpc_transport_unref (state->xprt); + state->xprt = NULL; + } if (state->fd) { fd_unref (state->fd); state->fd = NULL; @@ -776,7 +780,7 @@ server_alloc_frame (rpcsvc_request_t *req) if (conn->bound_xl) state->itable = conn->bound_xl->itable; - state->xprt = req->trans; + state->xprt = rpc_transport_ref (req->trans); state->conn = conn; state->resolve.fd_no = -1; diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c index 2851f8cea..4fcce6e99 100644 --- a/xlators/protocol/server/src/server.c +++ b/xlators/protocol/server/src/server.c @@ -424,11 +424,16 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event, case RPCSVC_EVENT_DISCONNECT: conn = get_server_conn_state (this, xprt); if (conn) - server_connection_put (this, conn); + server_connection_cleanup (this, conn); list_del (&xprt->list); break; + case RPCSVC_EVENT_TRANSPORT_DESTROY: + conn = get_server_conn_state (this, xprt); + if (conn) + server_connection_put (this, conn); + break; default: break; } -- cgit