epoll: edge triggered and multi-threaded epoll

- edge triggered (oneshot) polling with epoll
- pick one event to avoid multiple events getting picked up by same
  thread
  and so get better distribution of events against multiple threads
- wire support for multiple poll threads to epoll_wait in parallel
- evdata to store absolute index and not hint for epoll
- store index and gen of slot instead of fd and index hint
- perform fd close asynchronously inside event.c for multithread safety
- poll is still single threaded

Change-Id: I536851dda0ab224c5d5a1b130a571397c9cace8f
BUG: 1104462
Signed-off-by: Anand Avati <avati@redhat.com>
Signed-off-by: Vijaikumar M <vmallika@redhat.com>
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Signed-off-by: Shyam <srangana@redhat.com>
Reviewed-on: http://review.gluster.org/3842
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 28 insertions, 5 deletions

diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index 107590b0273..0a3a5812a91 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -291,6 +291,22 @@ ssl_do (rpc_transport_t *this, void *buf, size_t len, SSL_trinary_func *func)
 		case SSL_ERROR_NONE:
 			return r;
 		case SSL_ERROR_WANT_READ:
+                        /* If we are attempting to connect/accept then we
+                         * should wait here on the poll, for the SSL
+                         * (re)negotiation to complete, else we would error out
+                         * on the accept/connect.
+                         * If we are here when attempting to read/write
+                         * then we return r (or -1) as the socket is always
+                         * primed for the read event, and it would eventually
+                         * call one of the SSL routines */
+                        /* NOTE: Only way to determine this is a accept/connect
+                         * is to examine buf or func, which is not very
+                         * clean */
+                        if ((func == (SSL_trinary_func *)SSL_read)
+                            || (func == (SSL_trinary_func *) SSL_write)) {
+                                return r;
+                        }
+
 			pfd.fd = priv->sock;
 			pfd.events = POLLIN;
 			if (poll(&pfd,1,-1) < 0) {
@@ -944,9 +960,8 @@ __socket_reset (rpc_transport_t *this)
 
         memset (&priv->incoming, 0, sizeof (priv->incoming));
 
-        event_unregister (this->ctx->event_pool, priv->sock, priv->idx);
+        event_unregister_close (this->ctx->event_pool, priv->sock, priv->idx);
 
-        close (priv->sock);
         priv->sock = -1;
         priv->idx = -1;
         priv->connected = -1;
@@ -2236,9 +2251,16 @@ socket_event_poll_in (rpc_transport_t *this)
         rpc_transport_pollin_t *pollin = NULL;
         socket_private_t       *priv = this->private;
 
-        ret = socket_proto_state_machine (this, &pollin);
+	do {
+		/* consume all we can, this is our only chance
+		   (Edge Triggered polling in epoll)
+		*/
+		pollin = NULL;
+		ret = socket_proto_state_machine (this, &pollin);
+
+		if (!pollin)
+			break;
 
-        if (pollin != NULL) {
                 priv->ot_state = OT_CALLBACK;
                 ret = rpc_transport_notify (this, RPC_TRANSPORT_MSG_RECEIVED,
                                             pollin);
@@ -2246,7 +2268,8 @@ socket_event_poll_in (rpc_transport_t *this)
                         priv->ot_state = OT_RUNNING;
                 }
                 rpc_transport_pollin_destroy (pollin);
-        }
+
+        } while (pollin);
 
         return ret;
 }