From 0a20e56d07de3e467e09da885a6b71cdc165de17 Mon Sep 17 00:00:00 2001 From: Mohammed Rafi KC Date: Mon, 29 May 2017 16:00:24 +0530 Subject: protocol/server: make listen backlog value as configurable problem: When we call listen from protocol/server, we are giving a hard coded valie of 10 if it is not manually given. With multiplexing, especially when glusterd restarts all clients may try to connect to the server at a time. Which will result in overflowing the queue, and kernel will complain about the errors. Solution: This patch will introduce a volume set command to make backlog value as a configurable. This patch also changes the default values for backlog from 10 to 128. This changes is only applicable for sockets listening from protocol. Example: gluster volume set transport.listen-backlog 1024 Note: 1 Brick has to be restarted to get this value in effect 2 This changes won't be reflected in glusterd, or other xlators which calls listen. If you need, you have to add this option to the volfile. Change-Id: I0c5a2bbf28b5db612f9979e7560e05dd82b41477 BUG: 1456405 Signed-off-by: Mohammed Rafi KC Reviewed-on: https://review.gluster.org/17411 Smoke: Gluster Build System NetBSD-regression: NetBSD Build System CentOS-regression: Gluster Build System Reviewed-by: Raghavendra G Reviewed-by: Raghavendra Talur Reviewed-by: Atin Mukherjee Reviewed-by: Niels de Vos Reviewed-by: Jeff Darcy --- rpc/rpc-transport/rdma/src/rdma.c | 32 +++++++++++++++++++++++++++++++- rpc/rpc-transport/rdma/src/rdma.h | 2 ++ rpc/rpc-transport/socket/src/socket.c | 24 ++++++++++++++---------- 3 files changed, 47 insertions(+), 11 deletions(-) (limited to 'rpc') diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c index 8d9e6474f3c..01f96c21b5c 100644 --- a/rpc/rpc-transport/rdma/src/rdma.c +++ b/rpc/rpc-transport/rdma/src/rdma.c @@ -4499,6 +4499,12 @@ gf_rdma_options_init (rpc_transport_t *this) options->attr_retry_cnt = GF_RDMA_RETRY_CNT; options->attr_rnr_retry = GF_RDMA_RNR_RETRY; + temp = dict_get (this->options, "transport.listen-backlog"); + if (temp) + options->backlog = data_to_uint32 (temp); + else + options->backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG; + temp = dict_get (this->options, "transport.rdma.work-request-send-count"); if (temp) @@ -4635,6 +4641,7 @@ gf_rdma_init (rpc_transport_t *this) priv->peer.recv_count = options->recv_count; priv->peer.send_size = options->send_size; priv->peer.recv_size = options->recv_size; + priv->backlog = options->backlog; priv->peer.trans = this; INIT_LIST_HEAD (&priv->peer.ioq); @@ -4848,7 +4855,8 @@ gf_rdma_listen (rpc_transport_t *this) goto err; } - ret = rdma_listen (peer->cm_id, 10); + ret = rdma_listen (peer->cm_id, priv->backlog); + if (ret != 0) { gf_msg (this->name, GF_LOG_WARNING, errno, RDMA_MSG_LISTEN_FAILED, @@ -4919,6 +4927,28 @@ init (rpc_transport_t *this) return 0; } +int +reconfigure (rpc_transport_t *this, dict_t *options) +{ + gf_rdma_private_t *priv = NULL; + uint32_t backlog = 0; + int ret = -1; + + GF_VALIDATE_OR_GOTO ("rdma", this, out); + GF_VALIDATE_OR_GOTO ("rdma", this->private, out); + + priv = this->private; + + if (dict_get_uint32 (options, "transport.listen-backlog", + &backlog) == 0) { + priv->backlog = backlog; + gf_log (this->name, GF_LOG_DEBUG, "Reconfigued " + "transport.listen-backlog=%d", priv->backlog); + } + ret = 0; +out: + return ret; +} void fini (struct rpc_transport *this) { diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h index 449861f075f..6bcf6bc6ef2 100644 --- a/rpc/rpc-transport/rdma/src/rdma.h +++ b/rpc/rpc-transport/rdma/src/rdma.h @@ -144,6 +144,7 @@ struct __gf_rdma_options { uint8_t attr_timeout; uint8_t attr_retry_cnt; uint8_t attr_rnr_retry; + uint32_t backlog; }; typedef struct __gf_rdma_options gf_rdma_options_t; @@ -383,6 +384,7 @@ struct __gf_rdma_private { pthread_mutex_t recv_mutex; pthread_cond_t recv_cond; gf_rdma_transport_entity_t entity; + uint32_t backlog; }; typedef struct __gf_rdma_private gf_rdma_private_t; diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c index f5062fb026c..dd7f5d3b77e 100644 --- a/rpc/rpc-transport/socket/src/socket.c +++ b/rpc/rpc-transport/socket/src/socket.c @@ -3536,10 +3536,7 @@ socket_listen (rpc_transport_t *this) goto unlock; } - if (priv->backlog) - ret = listen (priv->sock, priv->backlog); - else - ret = listen (priv->sock, 10); + ret = listen (priv->sock, priv->backlog); if (ret == -1) { gf_log (this->name, GF_LOG_ERROR, @@ -3853,6 +3850,7 @@ reconfigure (rpc_transport_t *this, dict_t *options) gf_boolean_t tmp_bool = _gf_false; char *optstr = NULL; int ret = 0; + uint32_t backlog = 0; uint64_t windowsize = 0; uint32_t timeout = 0; int keepaliveidle = GF_KEEPALIVE_TIME; @@ -3892,6 +3890,13 @@ reconfigure (rpc_transport_t *this, dict_t *options) gf_log (this->name, GF_LOG_DEBUG, "Reconfigued " "transport.tcp-user-timeout=%d", priv->timeout); + if (dict_get_uint32 (options, "transport.listen-backlog", + &backlog) == 0) { + priv->backlog = backlog; + gf_log (this->name, GF_LOG_DEBUG, "Reconfigued " + "transport.listen-backlog=%d", priv->backlog); + } + if (dict_get_int32 (options, "transport.socket.keepalive-time", &(priv->keepaliveidle)) != 0) priv->keepaliveidle = keepaliveidle; @@ -4190,10 +4195,12 @@ socket_init (rpc_transport_t *this) "transport.keepalivecnt=%d", keepalivecnt); if (dict_get_uint32 (this->options, - "transport.socket.listen-backlog", - &backlog) == 0) { - priv->backlog = backlog; + "transport.listen-backlog", + &backlog) != 0) { + + backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG; } + priv->backlog = backlog; optstr = NULL; @@ -4612,9 +4619,6 @@ struct volume_options options[] = { .type = GF_OPTION_TYPE_INT, .default_value = "9" }, - { .key = {"transport.socket.listen-backlog"}, - .type = GF_OPTION_TYPE_INT - }, { .key = {"transport.socket.read-fail-log"}, .type = GF_OPTION_TYPE_BOOL }, -- cgit