protocol/rpc/transport: bring in one more event for 'TRANSPORT-DESTROY'

needed because, a RPC disconnect doesn't mean that a RPC transport/listener
is dead. With this, the race in server protocol cleaning up the lock table /
fd table when some frames are in transit will be handled properly.

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1843 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1843

7 files changed, 34 insertions, 29 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 61bc5263dc7..bd878f4fc1e 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1428,32 +1428,16 @@ rpc_clnt_ref (struct rpc_clnt *rpc)
         return rpc;
 }
 
-struct rpc_clnt *
-rpc_clnt_unref (struct rpc_clnt *rpc)
-{
-        int     count = 0;
-
-        if (!rpc)
-                return NULL;
-        pthread_mutex_lock (&rpc->lock);
-        {
-                count = --rpc->refcount;
-        }
-        pthread_mutex_unlock (&rpc->lock);
-        if (!count) {
-                rpc_clnt_destroy (rpc);
-                return NULL;
-        }
-        return rpc;
-}
 
-void
+static void
 rpc_clnt_destroy (struct rpc_clnt *rpc)
 {
         if (!rpc)
                 return;
 
-        rpc_transport_destroy (rpc->conn.trans);
+        if (rpc->conn.trans)
+                rpc_transport_unref (rpc->conn.trans);
+
         rpc_clnt_connection_cleanup (&rpc->conn);
         rpc_clnt_reconnect_cleanup (&rpc->conn);
         saved_frames_destroy (rpc->conn.saved_frames);
@@ -1469,6 +1453,25 @@ rpc_clnt_destroy (struct rpc_clnt *rpc)
         return;
 }
 
+struct rpc_clnt *
+rpc_clnt_unref (struct rpc_clnt *rpc)
+{
+        int     count = 0;
+
+        if (!rpc)
+                return NULL;
+        pthread_mutex_lock (&rpc->lock);
+        {
+                count = --rpc->refcount;
+        }
+        pthread_mutex_unlock (&rpc->lock);
+        if (!count) {
+                rpc_clnt_destroy (rpc);
+                return NULL;
+        }
+        return rpc;
+}
+
 
 void
 rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index f5fe17f5a47..932497b1387 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -223,8 +223,6 @@ rpc_clnt_ref (struct rpc_clnt *rpc);
 struct rpc_clnt *
 rpc_clnt_unref (struct rpc_clnt *rpc);
 
-void rpc_clnt_destroy (struct rpc_clnt *rpc);
-
 void rpc_clnt_set_connected (rpc_clnt_connection_t *conn);
 
 void rpc_clnt_unset_connected (rpc_clnt_connection_t *conn);
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index c69237d58c4..43b2a0c18cf 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -1110,8 +1110,7 @@ rpc_transport_unref (rpc_transport_t *this)
 	pthread_mutex_unlock (&this->lock);
 
 	if (refcount == 0) {
-		/* xlator_notify (this->xl, GF_EVENT_RPC_TRANSPORT_CLEANUP,
-                   this); */
+		this->notify (this, this->mydata, RPC_TRANSPORT_CLEANUP, NULL);
 		rpc_transport_destroy (this);
 	}
 
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 2ba46fba9f7..478de9ef1c4 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -248,9 +248,6 @@ int32_t
 rpc_transport_submit_reply (rpc_transport_t *this,
                             rpc_transport_reply_t *reply);
 
-int32_t
-rpc_transport_destroy (rpc_transport_t *this);
-
 rpc_transport_t *
 rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *name);
 
diff --git a/rpc/rpc-lib/src/rpcsvc-common.h b/rpc/rpc-lib/src/rpcsvc-common.h
index 7e72bc3ae44..442049433a7 100644
--- a/rpc/rpc-lib/src/rpcsvc-common.h
+++ b/rpc/rpc-lib/src/rpcsvc-common.h
@@ -29,6 +29,7 @@
 typedef enum {
         RPCSVC_EVENT_ACCEPT,
         RPCSVC_EVENT_DISCONNECT,
+        RPCSVC_EVENT_TRANSPORT_DESTROY,
         RPCSVC_EVENT_LISTENER_DEAD,
 } rpcsvc_event_t;
 
diff --git a/rpc/rpc-lib/src/rpcsvc.c b/rpc/rpc-lib/src/rpcsvc.c
index b78b1f0e2c9..e14313fe74f 100644
--- a/rpc/rpc-lib/src/rpcsvc.c
+++ b/rpc/rpc-lib/src/rpcsvc.c
@@ -1066,6 +1066,7 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
         rpc_transport_pollin_t *msg       = NULL;
         rpc_transport_t        *new_trans = NULL;
         rpcsvc_t               *svc       = NULL;
+        rpcsvc_listener_t      *listener  = NULL;
 
         svc = mydata;
         if (svc == NULL) {
@@ -1101,7 +1102,13 @@ rpcsvc_notify (rpc_transport_t *trans, void *mydata,
                 break;
 
         case RPC_TRANSPORT_CLEANUP:
-                /* FIXME: think about this later */
+                listener = rpcsvc_get_listener (svc, -1, trans);
+                if (listener == NULL) {
+                        goto out;
+                }
+
+                rpcsvc_program_notify (listener, RPCSVC_EVENT_TRANSPORT_DESTROY,
+                                       trans);
                 ret = 0;
                 break;
 
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index ae4bd56a821..40531ad4c2a 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -1736,7 +1736,7 @@ socket_event_handler (int fd, int idx, void *data,
         if ((ret < 0) || poll_err) {
                 gf_log ("transport", GF_LOG_TRACE, "disconnecting now");
                 socket_event_poll_err (this);
-                //rpc_transport_unref (this);
+                rpc_transport_unref (this);
         }
 
 out: