summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMohammed Rafi KC <rkavunga@redhat.com>2014-10-16 11:28:33 +0530
committerRaghavendra G <rgowdapp@redhat.com>2014-11-18 00:50:13 -0800
commit85e92d160bd71fdaeb3ae89d3440eec8fd438da9 (patch)
treefffe8616d07ff4e0020a2e2737bf5c0529e26dc2
parent43800dedb5f0b4644de913c3fd50c6b409ec7210 (diff)
rdma: client connection establishment takes more time
For rdma type only volume client connection establishment with server takes more than three seconds. Because for tcp,rdma type volume, will have 2 ports one for tcp and one for rdma, tcp port is stored with brickname and rdma port is stored as "brickname.rdma" during pamap_sighin. During the handshake when trying to get the brick port for rdma clients, since we are not aware of server transport type, we will append '.rdma' with brick name. So for tcp,rdma volume there will be an entry with '.rdma', but it will fail for rdma type only volume. So we will try again, this time without appending '.rdma' using a flag variable need_different_port, and it will succeed, but the reconnection happens only after 3 seconds. In this patch for rdma only type volume we will append '.rdma' during the pmap_signin. So during the handshake we will get the correct port for first try itself. Since we don't need to retry , we can remove the need_different_port flag variable. Change-Id: Ie8e3a7f532d4104829dbe995e99b35e95571466c BUG: 1153569 Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com> Reviewed-on: http://review.gluster.org/8934 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Krishnan Parthasarathi <kparthas@redhat.com> Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Tested-by: Raghavendra G <rgowdapp@redhat.com>
-rw-r--r--glusterfsd/src/glusterfsd-mgmt.c28
-rw-r--r--rpc/rpc-transport/rdma/src/rdma.c2
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-replace-brick.c27
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c6
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-ops.c22
-rw-r--r--xlators/protocol/client/src/client-handshake.c19
-rw-r--r--xlators/protocol/client/src/client.h4
7 files changed, 65 insertions, 43 deletions
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 9addd77cb26..3ff3337c01d 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -2160,10 +2160,11 @@ out:
int
glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
{
- call_frame_t *frame = NULL;
- pmap_signin_req req = {0, };
- int ret = -1;
- cmd_args_t *cmd_args = NULL;
+ call_frame_t *frame = NULL;
+ pmap_signin_req req = {0, };
+ int ret = -1;
+ cmd_args_t *cmd_args = NULL;
+ char brick_name[PATH_MAX] = {0,};
frame = create_frame (THIS, ctx->pool);
cmd_args = &ctx->cmd_args;
@@ -2174,8 +2175,15 @@ glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx)
goto out;
}
+ if (cmd_args->volfile_server_transport &&
+ !strcmp(cmd_args->volfile_server_transport, "rdma")) {
+ snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+ cmd_args->brick_name);
+ req.brick = brick_name;
+ } else
+ req.brick = cmd_args->brick_name;
+
req.port = cmd_args->brick_port;
- req.brick = cmd_args->brick_name;
ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
GF_PMAP_SIGNIN, mgmt_pmap_signin_cbk,
@@ -2226,6 +2234,7 @@ glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
pmap_signout_req req = {0, };
call_frame_t *frame = NULL;
cmd_args_t *cmd_args = NULL;
+ char brick_name[PATH_MAX] = {0,};
frame = create_frame (THIS, ctx->pool);
cmd_args = &ctx->cmd_args;
@@ -2236,8 +2245,15 @@ glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx)
goto out;
}
+ if (cmd_args->volfile_server_transport &&
+ !strcmp(cmd_args->volfile_server_transport, "rdma")) {
+ snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+ cmd_args->brick_name);
+ req.brick = brick_name;
+ } else
+ req.brick = cmd_args->brick_name;
+
req.port = cmd_args->brick_port;
- req.brick = cmd_args->brick_name;
req.rdma_port = cmd_args->brick_port2;
ret = mgmt_submit_request (&req, frame, ctx, &clnt_pmap_prog,
GF_PMAP_SIGNOUT, mgmt_pmap_signout_cbk,
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 89ed4558d72..1e0cce54b7a 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -4274,7 +4274,7 @@ gf_rdma_disconnect (rpc_transport_t *this)
int32_t ret = 0;
priv = this->private;
- gf_log_callingfn (this->name, GF_LOG_WARNING,
+ gf_log_callingfn (this->name, GF_LOG_DEBUG,
"disconnect called (peer:%s)",
this->peerinfo.identifier);
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 8bfa2d4bdd6..5a98d497137 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -1390,14 +1390,16 @@ umount:
* after all commit acks are received.
*/
static int
-rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
- dict_t *req_dict, int32_t replace_op)
+rb_update_srcbrick_port (glusterd_volinfo_t *volinfo,
+ glusterd_brickinfo_t *src_brickinfo,
+ dict_t *rsp_dict, dict_t *req_dict, int32_t replace_op)
{
- xlator_t *this = NULL;
- dict_t *ctx = NULL;
- int ret = 0;
- int dict_ret = 0;
- int src_port = 0;
+ xlator_t *this = NULL;
+ dict_t *ctx = NULL;
+ int ret = 0;
+ int dict_ret = 0;
+ int src_port = 0;
+ char brickname[PATH_MAX] = {0,};
this = THIS;
@@ -1409,8 +1411,15 @@ rb_update_srcbrick_port (glusterd_brickinfo_t *src_brickinfo, dict_t *rsp_dict,
gf_log ("", GF_LOG_INFO,
"adding src-brick port no");
+ if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+ snprintf (brickname, sizeof(brickname), "%s.rdma",
+ src_brickinfo->path);
+ } else
+ snprintf (brickname, sizeof(brickname), "%s",
+ src_brickinfo->path);
+
src_brickinfo->port = pmap_registry_search (this,
- src_brickinfo->path, GF_PMAP_PORT_BRICKSERVER);
+ brickname, GF_PMAP_PORT_BRICKSERVER);
if (!src_brickinfo->port &&
replace_op != GF_REPLACE_OP_COMMIT_FORCE ) {
gf_log ("", GF_LOG_ERROR,
@@ -1645,7 +1654,7 @@ glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
goto out;
}
- ret = rb_update_srcbrick_port (src_brickinfo, rsp_dict,
+ ret = rb_update_srcbrick_port (volinfo, src_brickinfo, rsp_dict,
dict, replace_op);
if (ret)
goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index bb9fa45d24a..ad824be73f3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1968,6 +1968,12 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t *volinfo,
bind_address);
}
+ if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+ runner_argprintf (&runner, "--volfile-server-transport=rdma");
+ else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+ runner_argprintf (&runner,
+ "--volfile-server-transport=socket,rdma");
+
if (volinfo->memory_accounting)
runner_add_arg (&runner, "--mem-accounting");
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 4685fa6a8e4..dd3308c40d2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2583,12 +2583,13 @@ int
glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
char **xl_opts)
{
- glusterd_brickinfo_t *brickinfo = NULL;
- glusterd_conf_t *priv = NULL;
- int index = 0;
- int ret = -1;
- int i = 0;
- int port = 0;
+ glusterd_brickinfo_t *brickinfo = NULL;
+ glusterd_conf_t *priv = NULL;
+ char brickname[PATH_MAX] = {0,};
+ int index = 0;
+ int ret = -1;
+ int i = 0;
+ int port = 0;
GF_ASSERT (xl_opts);
if (!xl_opts) {
@@ -2605,7 +2606,14 @@ glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
if (uuid_compare (brickinfo->uuid, MY_UUID))
continue;
- port = pmap_registry_search (THIS, brickinfo->path,
+ if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+ snprintf (brickname, sizeof(brickname), "%s.rdma",
+ brickinfo->path);
+ } else
+ snprintf (brickname, sizeof(brickname), "%s",
+ brickinfo->path);
+
+ port = pmap_registry_search (THIS, brickname,
GF_PMAP_PORT_BRICKSERVER);
if (!port) {
ret = -1;
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index 8bd3d6a3d09..42b7ac0745e 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -1205,8 +1205,6 @@ client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *m
conf->connecting = 0;
conf->connected = 1;
- conf->need_different_port = 0;
-
if (lk_ver != client_get_lk_ver (conf)) {
gf_log (this->name, GF_LOG_INFO, "Server and Client "
"lk-version numbers are not same, reopening the fds");
@@ -1546,22 +1544,11 @@ client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
req.brick = remote_subvol;
- /* FIXME: Dirty work around */
if (!dict_get_str (options, "transport-type", &xprt)) {
- /* This logic is required only in case of 'rdma' client
- transport-type and the volume is of 'tcp,rdma'
- transport type. */
if (!strcmp (xprt, "rdma")) {
- if (!conf->need_different_port) {
- snprintf (brick_name, PATH_MAX, "%s.rdma",
- remote_subvol);
- req.brick = brick_name;
- conf->need_different_port = 1;
- conf->skip_notify = 1;
- } else {
- conf->need_different_port = 0;
- conf->skip_notify = 0;
- }
+ snprintf (brick_name, sizeof(brick_name), "%s.rdma",
+ remote_subvol);
+ req.brick = brick_name;
}
}
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 7f7d511910e..69f77570cd0 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -96,10 +96,6 @@ typedef struct clnt_conf {
char disconnect_err_logged; /* flag used to prevent
excessive disconnect
logging */
-
- char need_different_port; /* flag used to change the
- portmap path in case of
- 'tcp,rdma' on server */
gf_boolean_t lk_heal;
uint16_t lk_version; /* this variable is used to distinguish
client-server transaction while