summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAmar Tumballi <amar@gluster.com>2010-10-07 05:56:22 +0000
committerVijay Bellur <vijay@dev.gluster.com>2010-10-07 05:09:08 -0700
commit5979bc3b964adcb8a536dc354e4eca08acaff03e (patch)
treeca1a5ed41fbc95bd51be7791ae0445132b1593db
parent3c0a7fb86eef1eecada77a6f040f96e1fd9e0536 (diff)
protocol/rpc/transport: bring in one more event for 'TRANSPORT-DESTROY'
needed because, a RPC disconnect doesn't mean that a RPC transport/listener is dead. With this, the race in server protocol cleaning up the lock table / fd table when some frames are in transit will be handled properly. Signed-off-by: Amar Tumballi <amar@gluster.com> Signed-off-by: Vijay Bellur <vijay@dev.gluster.com> BUG: 1843 () URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1843
-rw-r--r--glusterfsd/src/glusterfsd.c2
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.c43
-rw-r--r--rpc/rpc-lib/src/rpc-clnt.h2
-rw-r--r--rpc/rpc-lib/src/rpc-transport.c3
-rw-r--r--rpc/rpc-lib/src/rpc-transport.h3
-rw-r--r--rpc/rpc-lib/src/rpcsvc-common.h1
-rw-r--r--rpc/rpc-lib/src/rpcsvc.c9
-rw-r--r--rpc/rpc-transport/socket/src/socket.c2
-rw-r--r--xlators/protocol/client/src/client.c2
-rw-r--r--xlators/protocol/server/src/server-helpers.c6
-rw-r--r--xlators/protocol/server/src/server.c7
11 files changed, 47 insertions, 33 deletions
diff --git a/glusterfsd/src/glusterfsd.c b/glusterfsd/src/glusterfsd.c
index eb2f61ba577..4d8eb22fe0c 100644
--- a/glusterfsd/src/glusterfsd.c
+++ b/glusterfsd/src/glusterfsd.c
@@ -661,7 +661,7 @@ cleanup_and_exit (int signum)
glusterfs_mgmt_pmap_signout (ctx);
if (ctx->mgmt)
- rpc_clnt_destroy (ctx->mgmt);
+ rpc_clnt_unref (ctx->mgmt);
gf_log ("glusterfsd", GF_LOG_NORMAL, "shutting down");
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 61bc5263dc7..bd878f4fc1e 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1428,32 +1428,16 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
return rpc;
}
-struct rpc_clnt *
-rpc_clnt_unref (struct rpc_clnt *rpc)
-{
- int count = 0;
-
- if (!rpc)
- return NULL;
- pthread_mutex_lock (&rpc->lock);
- {
- count = --rpc->refcount;
- }
- pthread_mutex_unlock (&rpc->lock);
- if (!count) {
- rpc_clnt_destroy (rpc);
- return NULL;
- }
- return rpc;
-}
-void
+static void
rpc_clnt_destroy (struct rpc_clnt *rpc)
{
if (!rpc)
return;
- rpc_transport_destroy (rpc->conn.trans);
+ if (rpc->conn.trans)
+ rpc_transport_unref (rpc->conn.trans);
+
rpc_clnt_connection_cleanup (&rpc->conn);
rpc_clnt_reconnect_cleanup (&rpc->conn);
saved_frames_destroy (rpc->conn.saved_frames);
@@ -1469,6 +1453,25 @@ rpc_clnt_destroy (struct rpc_clnt *rpc)
return;
}
+struct rpc_clnt *
+rpc_clnt_unref (struct rpc_clnt *rpc)
+{
+ int count = 0;
+
+ if (!rpc)
+ return NULL;
+ pthread_mutex_lock (&rpc->lock);
+ {
+ count = --rpc->refcount;
+ }
+ pthread_mutex_unlock (&rpc->lock);
+ if (!count) {
+ rpc_clnt_destroy (rpc);
+ return NULL;
+ }
+ return rpc;
+}
+
void
rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index f5fe17f5a47..932497b1387 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -223,8 +223,6 @@ rpc_clnt_ref (struct rpc_clnt *rpc);
struct rpc_clnt *
rpc_clnt_unref (struct rpc_clnt *rpc);
-void rpc_clnt_destroy (struct rpc_clnt *rpc);
-
void rpc_clnt_set_connected (rpc_clnt_connection_t *conn);
void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn);
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index c69237d58c4..43b2a0c18cf 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -1110,8 +1110,7 @@ rpc_transport_unref (rpc_transport_t *this)
pthread_mutex_unlock (&this->lock);
if (refcount == 0) {
- /* xlator_notify (this->xl, GF_EVENT_RPC_TRANSPORT_CLEANUP,
- this); */
+ this->notify (this, this->mydata, RPC_TRANSPORT_CLEANUP, NULL);
rpc_transport_destroy (this);
}
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 2ba46fba9f7..478de9ef1c4 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -248,9 +248,6 @@ int32_t
rpc_transport_submit_reply (rpc_transport_t *this,
rpc_transport_reply_t *reply);
-int32_t
-rpc_transport_destroy (rpc_transport_t *this);
-
rpc_transport_t *
rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *name);
diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h
index 7e72bc3ae44..442049433a7 100644
--- a/rpc/rpc-lib/src/rpcsvc-common.h
+++ b/rpc/rpc-lib/src/rpcsvc-common.h
@@ -29,6 +29,7 @@
typedef enum {
RPCSVC_EVENT_ACCEPT,
RPCSVC_EVENT_DISCONNECT,
+ RPCSVC_EVENT_TRANSPORT_DESTROY,
RPCSVC_EVENT_LISTENER_DEAD,
} rpcsvc_event_t;
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index b78b1f0e2c9..e14313fe74f 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1066,6 +1066,7 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
rpc_transport_pollin_t *msg = NULL;
rpc_transport_t *new_trans = NULL;
rpcsvc_t *svc = NULL;
+ rpcsvc_listener_t *listener = NULL;
svc = mydata;
if (svc == NULL) {
@@ -1101,7 +1102,13 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
break;
case RPC_TRANSPORT_CLEANUP:
- /* FIXME: think about this later */
+ listener = rpcsvc_get_listener (svc, -1, trans);
+ if (listener == NULL) {
+ goto out;
+ }
+
+ rpcsvc_program_notify (listener, RPCSVC_EVENT_TRANSPORT_DESTROY,
+ trans);
ret = 0;
break;
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index ae4bd56a821..40531ad4c2a 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -1736,7 +1736,7 @@ socket_event_handler (int fd, int idx, void *data,
if ((ret < 0) || poll_err) {
gf_log ("transport", GF_LOG_TRACE, "disconnecting now");
socket_event_poll_err (this);
- //rpc_transport_unref (this);
+ rpc_transport_unref (this);
}
out:
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 50cb917aacc..1cc96532b04 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2008,7 +2008,7 @@ fini (xlator_t *this)
if (conf) {
if (conf->rpc)
- rpc_clnt_destroy (conf->rpc);
+ rpc_clnt_unref (conf->rpc);
/* Saved Fds */
/* TODO: */
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 5ea6f561908..69fb5f6a7d6 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -102,6 +102,10 @@ free_state (server_state_t *state)
state->conn = NULL;
}
+ if (state->xprt) {
+ rpc_transport_unref (state->xprt);
+ state->xprt = NULL;
+ }
if (state->fd) {
fd_unref (state->fd);
state->fd = NULL;
@@ -776,7 +780,7 @@ server_alloc_frame (rpcsvc_request_t *req)
if (conn->bound_xl)
state->itable = conn->bound_xl->itable;
- state->xprt = req->trans;
+ state->xprt = rpc_transport_ref (req->trans);
state->conn = conn;
state->resolve.fd_no = -1;
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index 2851f8cea86..4fcce6e99d2 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -424,11 +424,16 @@ server_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
case RPCSVC_EVENT_DISCONNECT:
conn = get_server_conn_state (this, xprt);
if (conn)
- server_connection_put (this, conn);
+ server_connection_cleanup (this, conn);
list_del (&xprt->list);
break;
+ case RPCSVC_EVENT_TRANSPORT_DESTROY:
+ conn = get_server_conn_state (this, xprt);
+ if (conn)
+ server_connection_put (this, conn);
+ break;
default:
break;
}