diff options
author | Mohammed Rafi KC <rkavunga@redhat.com> | 2014-10-16 11:28:33 +0530 |
---|---|---|
committer | Raghavendra Bhat <raghavendra@redhat.com> | 2015-01-06 01:43:45 -0800 |
commit | 7efd84b1b743d3a91e23fd97dbf8a1d89b0d1f44 (patch) | |
tree | e7fdca416870f9a9471131cd5941fb4118d7eb20 /xlators | |
parent | bb1601b94039b27b5a148479b61895a100ce8e6d (diff) |
rdma: client connection establishment takes more time
Backport of http://review.gluster.org/8934
For rdma type only volume client connection establishment
with server takes more than three seconds. Because for
tcp,rdma type volume, will have 2 ports one for tcp and
one for rdma, tcp port is stored with brickname and rdma
port is stored as "brickname.rdma" during pamap_sighin.
During the handshake when trying to get the brick port
for rdma clients, since we are not aware of server
transport type, we will append '.rdma' with brick name.
So for tcp,rdma volume there will be an entry with
'.rdma', but it will fail for rdma type only volume.
So we will try again, this time without appending '.rdma'
using a flag variable need_different_port, and it will succeed,
but the reconnection happens only after 3 seconds.
In this patch for rdma only type volume
we will append '.rdma' during the pmap_signin. So during the
handshake we will get the correct port for first try itself.
Since we don't need to retry , we can remove the
need_different_port flag variable.
Change-Id: I82a8a27f0e65a2e287f321e5e8292d86c6baf5b4
BUG: 1166515
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/8934
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
Tested-by: Raghavendra G <rgowdapp@redhat.com>
Reviewed-on: http://review.gluster.org/9177
Reviewed-by: Raghavendra Bhat <raghavendra@redhat.com>
Diffstat (limited to 'xlators')
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-replace-brick.c | 27 | ||||
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-utils.c | 6 | ||||
-rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-volume-ops.c | 22 | ||||
-rw-r--r-- | xlators/protocol/client/src/client-handshake.c | 19 | ||||
-rw-r--r-- | xlators/protocol/client/src/client.h | 4 |
5 files changed, 42 insertions, 36 deletions
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c index 8bfa2d4bdd6..5a98d497137 100644 --- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c +++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c @@ -1390,14 +1390,16 @@ umount: * after all commit acks are received. */ static int -rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict, - dict_t *req_dict, int32_t replace_op) +rb_update_srcbrick_port (glusterd_volinfo_t *volinfo, + glusterd_brickinfo_t *src_brickinfo, + dict_t *rsp_dict, dict_t *req_dict, int32_t replace_op) { - xlator_t *this = NULL; - dict_t *ctx = NULL; - int ret = 0; - int dict_ret = 0; - int src_port = 0; + xlator_t *this = NULL; + dict_t *ctx = NULL; + int ret = 0; + int dict_ret = 0; + int src_port = 0; + char brickname[PATH_MAX] = {0,}; this = THIS; @@ -1409,8 +1411,15 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict, gf_log ("", GF_LOG_INFO, "adding src-brick port no"); + if (volinfo->transport_type == GF_TRANSPORT_RDMA) { + snprintf (brickname, sizeof(brickname), "%s.rdma", + src_brickinfo->path); + } else + snprintf (brickname, sizeof(brickname), "%s", + src_brickinfo->path); + src_brickinfo->port = pmap_registry_search (this, - src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER); + brickname, GF_PMAP_PORT_BRICKSERVER); if (!src_brickinfo->port && replace_op != GF_REPLACE_OP_COMMIT_FORCE ) { gf_log ("", GF_LOG_ERROR, @@ -1645,7 +1654,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict) goto out; } - ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict, + ret = rb_update_srcbrick_port (volinfo, src_brickinfo, rsp_dict, dict, replace_op); if (ret) goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index d2eab044dc6..fcaa215b578 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -1952,6 +1952,12 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo, bind_address); } + if (volinfo->transport_type == GF_TRANSPORT_RDMA) + runner_argprintf (&runner, "--volfile-server-transport=rdma"); + else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) + runner_argprintf (&runner, + "--volfile-server-transport=socket,rdma"); + if (volinfo->memory_accounting) runner_add_arg (&runner, "--mem-accounting"); diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c index b55275308e2..cb5d6f832f6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c +++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c @@ -2342,12 +2342,13 @@ int glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo, char **xl_opts) { - glusterd_brickinfo_t *brickinfo = NULL; - glusterd_conf_t *priv = NULL; - int index = 0; - int ret = -1; - int i = 0; - int port = 0; + glusterd_brickinfo_t *brickinfo = NULL; + glusterd_conf_t *priv = NULL; + char brickname[PATH_MAX] = {0,}; + int index = 0; + int ret = -1; + int i = 0; + int port = 0; GF_ASSERT (xl_opts); if (!xl_opts) { @@ -2364,7 +2365,14 @@ glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo, if (uuid_compare (brickinfo->uuid, MY_UUID)) continue; - port = pmap_registry_search (THIS, brickinfo->path, + if (volinfo->transport_type == GF_TRANSPORT_RDMA) { + snprintf (brickname, sizeof(brickname), "%s.rdma", + brickinfo->path); + } else + snprintf (brickname, sizeof(brickname), "%s", + brickinfo->path); + + port = pmap_registry_search (THIS, brickname, GF_PMAP_PORT_BRICKSERVER); if (!port) { ret = -1; diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c index 8bd3d6a3d09..42b7ac0745e 100644 --- a/xlators/protocol/client/src/client-handshake.c +++ b/xlators/protocol/client/src/client-handshake.c @@ -1205,8 +1205,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m conf->connecting = 0; conf->connected = 1; - conf->need_different_port = 0; - if (lk_ver != client_get_lk_ver (conf)) { gf_log (this->name, GF_LOG_INFO, "Server and Client " "lk-version numbers are not same, reopening the fds"); @@ -1546,22 +1544,11 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc) req.brick = remote_subvol; - /* FIXME: Dirty work around */ if (!dict_get_str (options, "transport-type", &xprt)) { - /* This logic is required only in case of 'rdma' client - transport-type and the volume is of 'tcp,rdma' - transport type. */ if (!strcmp (xprt, "rdma")) { - if (!conf->need_different_port) { - snprintf (brick_name, PATH_MAX, "%s.rdma", - remote_subvol); - req.brick = brick_name; - conf->need_different_port = 1; - conf->skip_notify = 1; - } else { - conf->need_different_port = 0; - conf->skip_notify = 0; - } + snprintf (brick_name, sizeof(brick_name), "%s.rdma", + remote_subvol); + req.brick = brick_name; } } diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h index 7f7d511910e..69f77570cd0 100644 --- a/xlators/protocol/client/src/client.h +++ b/xlators/protocol/client/src/client.h @@ -96,10 +96,6 @@ typedef struct clnt_conf { char disconnect_err_logged; /* flag used to prevent excessive disconnect logging */ - - char need_different_port; /* flag used to change the - portmap path in case of - 'tcp,rdma' on server */ gf_boolean_t lk_heal; uint16_t lk_version; /* this variable is used to distinguish client-server transaction while |