summaryrefslogtreecommitdiffstats
path: root/rpc/rpc-transport/socket/src/socket.c
diff options
context:
space:
mode:
authorRaghavendra Gowdappa <rgowdapp@redhat.com>2018-10-31 16:10:58 +0530
committerRaghavendra G <rgowdapp@redhat.com>2018-11-29 01:19:12 +0000
commit95e380eca19b9f0d03a53429535f15556e5724ad (patch)
treebe32fca4fbfa7d29e8571545af26e784d34e294c /rpc/rpc-transport/socket/src/socket.c
parentf0232d07f7e6543b56830be28f6e80f9085e6241 (diff)
rpcsvc: provide each request handler thread its own queue
A single global per program queue is contended by all request handler threads and event threads. This can lead to high contention. So, reduce the contention by providing each request handler thread its own private queue. Thanks to "Manoj Pillai"<mpillai@redhat.com> for the idea of pairing a single queue with a fixed request-handler-thread and event-thread, which brought down the performance regression due to overhead of queuing significantly. Thanks to "Xavi Hernandez"<xhernandez@redhat.com> for discussion on how to communicate the event-thread death to request-handler-thread. Thanks to "Karan Sandha"<ksandha@redhat.com> for voluntarily running the perf benchmarks to qualify that performance regression introduced by ping-timer-fixes is fixed with this patch and patiently running many iterations of regression tests while RCAing the issue. Thanks to "Milind Changire"<mchangir@redhat.com> for patiently running the many iterations of perf benchmarking tests while RCAing the regression caused by ping-timer-expiry fixes. Change-Id: I578c3fc67713f4234bd3abbec5d3fbba19059ea5 Fixes: bz#1644629 Signed-off-by: Raghavendra Gowdappa <rgowdapp@redhat.com>
Diffstat (limited to 'rpc/rpc-transport/socket/src/socket.c')
-rw-r--r--rpc/rpc-transport/socket/src/socket.c28
1 files changed, 21 insertions, 7 deletions
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index dc227137d57..776e647d4f6 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -2859,7 +2859,7 @@ socket_complete_connection(rpc_transport_t *this)
/* reads rpc_requests during pollin */
static int
socket_event_handler(int fd, int idx, int gen, void *data, int poll_in,
- int poll_out, int poll_err)
+ int poll_out, int poll_err, char event_thread_died)
{
rpc_transport_t *this = NULL;
socket_private_t *priv = NULL;
@@ -2869,6 +2869,11 @@ socket_event_handler(int fd, int idx, int gen, void *data, int poll_in,
this = data;
+ if (event_thread_died) {
+ /* to avoid duplicate notifications, notify only for listener sockets */
+ return 0;
+ }
+
GF_VALIDATE_OR_GOTO("socket", this, out);
GF_VALIDATE_OR_GOTO("socket", this->private, out);
GF_VALIDATE_OR_GOTO("socket", this->xl, out);
@@ -2967,7 +2972,7 @@ out:
static int
socket_server_event_handler(int fd, int idx, int gen, void *data, int poll_in,
- int poll_out, int poll_err)
+ int poll_out, int poll_err, char event_thread_died)
{
rpc_transport_t *this = NULL;
socket_private_t *priv = NULL;
@@ -2991,6 +2996,12 @@ socket_server_event_handler(int fd, int idx, int gen, void *data, int poll_in,
priv = this->private;
ctx = this->ctx;
+ if (event_thread_died) {
+ rpc_transport_notify(this, RPC_TRANSPORT_EVENT_THREAD_DIED,
+ (void *)(unsigned long)gen);
+ return 0;
+ }
+
/* NOTE:
* We have done away with the critical section in this function. since
* there's little that it helps with. There's no other code that
@@ -3099,6 +3110,7 @@ socket_server_event_handler(int fd, int idx, int gen, void *data, int poll_in,
new_trans->mydata = this->mydata;
new_trans->notify = this->notify;
new_trans->listener = this;
+ new_trans->notify_poller_death = this->poller_death_accept;
new_priv = new_trans->private;
if (new_sockaddr.ss_family == AF_UNIX) {
@@ -3149,9 +3161,9 @@ socket_server_event_handler(int fd, int idx, int gen, void *data, int poll_in,
ret = rpc_transport_notify(this, RPC_TRANSPORT_ACCEPT, new_trans);
if (ret != -1) {
- new_priv->idx = event_register(ctx->event_pool, new_sock,
- socket_event_handler, new_trans,
- 1, 0);
+ new_priv->idx = event_register(
+ ctx->event_pool, new_sock, socket_event_handler, new_trans,
+ 1, 0, new_trans->notify_poller_death);
if (new_priv->idx == -1) {
ret = -1;
gf_log(this->name, GF_LOG_ERROR,
@@ -3530,7 +3542,8 @@ socket_connect(rpc_transport_t *this, int port)
this->listener = this;
priv->idx = event_register(ctx->event_pool, priv->sock,
- socket_event_handler, this, 1, 1);
+ socket_event_handler, this, 1, 1,
+ this->notify_poller_death);
if (priv->idx == -1) {
gf_log("", GF_LOG_WARNING,
"failed to register the event; "
@@ -3709,7 +3722,8 @@ socket_listen(rpc_transport_t *this)
rpc_transport_ref(this);
priv->idx = event_register(ctx->event_pool, priv->sock,
- socket_server_event_handler, this, 1, 0);
+ socket_server_event_handler, this, 1, 0,
+ this->notify_poller_death);
if (priv->idx == -1) {
gf_log(this->name, GF_LOG_WARNING,