From 3dc782527482d2c62c9344669ccaa2dc42481a58 Mon Sep 17 00:00:00 2001 From: Pranith K Date: Mon, 11 Oct 2010 03:26:00 +0000 Subject: mgmt/glusterd: handle reqs from unknown peers for friend sm Signed-off-by: Pranith Kumar K Signed-off-by: Vijay Bellur BUG: 1838 (handle peer detach gracefully in case of lost frames) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1838 --- cli/src/cli3_1-cops.c | 11 ++ rpc/rpc-lib/src/protocol-common.h | 4 +- rpc/rpc-lib/src/rpc-clnt.c | 2 +- xlators/mgmt/glusterd/src/glusterd-handler.c | 145 +++++++++++++++++---------- xlators/mgmt/glusterd/src/glusterd-sm.c | 7 +- xlators/mgmt/glusterd/src/glusterd3_1-mops.c | 17 +++- 6 files changed, 126 insertions(+), 60 deletions(-) diff --git a/cli/src/cli3_1-cops.c b/cli/src/cli3_1-cops.c index 5b5d5425f15..ee66b165f4e 100644 --- a/cli/src/cli3_1-cops.c +++ b/cli/src/cli3_1-cops.c @@ -121,6 +121,17 @@ gf_cli3_1_probe_cbk (struct rpc_req *req, struct iovec *iov, "with existing volumes in the " "cluster", rsp.hostname); break; + case GF_PROBE_UNKNOWN_PEER: + cli_out ("%s responded with 'unknown peer' error, " + "this could happen if %s doesn't have" + " localhost in its peer database", + rsp.hostname, rsp.hostname); + break; + case GF_PROBE_ADD_FAILED: + cli_out ("Failed to add peer information " + "on %s" , rsp.hostname); + break; + default: cli_out ("Probe returned with unknown errno %d", rsp.op_errno); diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h index 40283bf427b..bfd043e31d2 100644 --- a/rpc/rpc-lib/src/protocol-common.h +++ b/rpc/rpc-lib/src/protocol-common.h @@ -161,7 +161,9 @@ enum gf_probe_resp { GF_PROBE_LOCALHOST, GF_PROBE_FRIEND, GF_PROBE_ANOTHER_CLUSTER, - GF_PROBE_VOLUME_CONFLICT + GF_PROBE_VOLUME_CONFLICT, + GF_PROBE_UNKNOWN_PEER, + GF_PROBE_ADD_FAILED }; enum gf_deprobe_resp { diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c index 8cf7f7444b0..a53ac4da95b 100644 --- a/rpc/rpc-lib/src/rpc-clnt.c +++ b/rpc/rpc-lib/src/rpc-clnt.c @@ -328,7 +328,7 @@ saved_frames_unwind (struct saved_frames *saved_frames) if (!trav->rpcreq || !trav->rpcreq->prog) continue; - gf_log ("rpc-clnt", GF_LOG_ERROR, + gf_log_callingfn ("rpc-clnt", GF_LOG_ERROR, "forced unwinding frame type(%s) op(%s(%d)) " "called at %s", trav->rpcreq->prog->progname, diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 49281b956d0..7d5b749cfe6 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -63,16 +63,17 @@ static int glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t uuid, - char *hostname, int port, dict_t *dict) + char *hostname, int port, + gd1_mgmt_friend_req *friend_req) { int ret = -1; glusterd_peerinfo_t *peerinfo = NULL; glusterd_friend_sm_event_t *event = NULL; glusterd_friend_req_ctx_t *ctx = NULL; - glusterd_peerctx_args_t args = {0}; char rhost[UNIX_PATH_MAX + 1] = {0}; uuid_t friend_uuid = {0}; char uuid_str[50] = {0,}; + dict_t *dict = NULL; uuid_unparse (uuid, uuid_str); uuid_parse (uuid_str, friend_uuid); @@ -83,15 +84,9 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t uuid, ret = glusterd_friend_find (uuid, rhost, &peerinfo); if (ret) { - gf_log ("glusterd", GF_LOG_NORMAL, - " for host: %s (%d)", rhost, port); - args.mode = GD_MODE_SWITCH_ON; - args.req = NULL; - ret = glusterd_friend_add (rhost, port, - GD_FRIEND_STATE_DEFAULT, - &friend_uuid, NULL, &peerinfo, 0, - &args); - + ret = glusterd_xfer_friend_add_resp (req, rhost, port, -1, + GF_PROBE_UNKNOWN_PEER); + goto out; } ret = glusterd_friend_sm_new_event @@ -116,8 +111,23 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t uuid, if (hostname) ctx->hostname = gf_strdup (hostname); ctx->req = req; - ctx->vols = dict; + dict = dict_new (); + if (!dict) { + ret = -1; + goto out; + } + + ret = dict_unserialize (friend_req->vols.vols_val, + friend_req->vols.vols_len, + &dict); + + if (ret) + goto out; + else + dict->extra_stdfree = friend_req->vols.vols_val; + + ctx->vols = dict; event->ctx = ctx; ret = glusterd_friend_sm_inject_event (event); @@ -134,14 +144,23 @@ out: if (0 != ret) { if (ctx && ctx->hostname) GF_FREE (ctx->hostname); - if (ctx && ctx->vols) - dict_destroy (ctx->vols); if (ctx) GF_FREE (ctx); + if (dict) { + if ((!dict->extra_stdfree) && + friend_req->vols.vols_val) + free (friend_req->vols.vols_val); + dict_unref (dict); + } else { + if (friend_req->vols.vols_val) + free (friend_req->vols.vols_val); + } + if (event) + GF_FREE (event); + } else { + if (peerinfo && (0 == peerinfo->connected)) + ret = GLUSTERD_CONNECTION_AWAITED; } - - if (0 == peerinfo->connected) - ret = GLUSTERD_CONNECTION_AWAITED; return ret; } @@ -161,9 +180,12 @@ glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t uuid, ret = glusterd_friend_find (uuid, hostname, &peerinfo); if (ret) { - gf_log ("glusterd", GF_LOG_NORMAL, - "Unable to find peer"); - + gf_log ("glusterd", GF_LOG_CRITICAL, + "Received remove-friend from unknown peer %s", + hostname); + ret = glusterd_xfer_friend_remove_resp (req, hostname, + port); + goto out; } ret = glusterd_friend_sm_new_event @@ -1919,6 +1941,8 @@ glusterd_op_stage_send_resp (rpcsvc_request_t *req, gf_log ("glusterd", GF_LOG_NORMAL, "Responded to stage, ret: %d", ret); + if (rsp.dict.dict_val) + GF_FREE (rsp.dict.dict_val); return ret; } @@ -1969,7 +1993,6 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req) int32_t ret = -1; gd1_mgmt_friend_req friend_req = {{0},}; char str[50] = {0,}; - dict_t *dict = NULL; GF_ASSERT (req); if (!gd_xdr_to_mgmt_friend_req (req->msg[0], &friend_req)) { @@ -1981,31 +2004,11 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req) gf_log ("glusterd", GF_LOG_NORMAL, "Received probe from uuid: %s", str); - - dict = dict_new (); - if (!dict) { - ret = -1; - goto out; - } - - ret = dict_unserialize (friend_req.vols.vols_val, - friend_req.vols.vols_len, - &dict); - - if (ret) - goto out; - else - dict->extra_stdfree = friend_req.vols.vols_val; - ret = glusterd_handle_friend_req (req, friend_req.uuid, friend_req.hostname, friend_req.port, - dict); + &friend_req); out: - if (GLUSTERD_CONNECTION_AWAITED != ret) { - if (ret && dict) - dict_unref (dict); - } if (friend_req.hostname) free (friend_req.hostname);//malloced by xdr @@ -2018,6 +2021,7 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req) int32_t ret = -1; gd1_mgmt_friend_req friend_req = {{0},}; char str[50]; + char remote_hostname[UNIX_PATH_MAX + 1] = {0,}; GF_ASSERT (req); if (!gd_xdr_to_mgmt_friend_req (req->msg[0], &friend_req)) { @@ -2030,8 +2034,14 @@ glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req) gf_log ("glusterd", GF_LOG_NORMAL, "Received unfriend from uuid: %s", str); + ret = glusterd_remote_hostname_get (req, remote_hostname, + sizeof (remote_hostname)); + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname"); + goto out; + } ret = glusterd_handle_unfriend_req (req, friend_req.uuid, - friend_req.hostname, friend_req.port); + remote_hostname, friend_req.port); out: if (friend_req.hostname) @@ -2097,6 +2107,12 @@ glusterd_handle_friend_update (rpcsvc_request_t *req) } uuid_unparse (friend_req.uuid, str); + ret = glusterd_friend_find (friend_req.uuid, NULL, &tmp); + if (ret) { + gf_log ("", GF_LOG_CRITICAL, "Received friend update request " + "from unknown peer %s", str); + goto out; + } gf_log ("glusterd", GF_LOG_NORMAL, "Received friend update from uuid: %s", str); @@ -2169,21 +2185,30 @@ out: uuid_copy (rsp.uuid, priv->uuid); ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL, gd_xdr_serialize_mgmt_friend_update_rsp); - if (dict) + if (dict) { + if (!dict->extra_stdfree && friend_req.friends.friends_val) + free (friend_req.friends.friends_val);//malloced by xdr dict_unref (dict); + } else { + if (friend_req.friends.friends_val) + free (friend_req.friends.friends_val);//malloced by xdr + } + return ret; } int glusterd_handle_probe_query (rpcsvc_request_t *req) { - int32_t ret = -1; - char str[50]; - xlator_t *this = NULL; - glusterd_conf_t *conf = NULL; - gd1_mgmt_probe_req probe_req = {{0},}; - gd1_mgmt_probe_rsp rsp = {{0},}; + int32_t ret = -1; + char str[50]; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + gd1_mgmt_probe_req probe_req = {{0},}; + gd1_mgmt_probe_rsp rsp = {{0},}; glusterd_peerinfo_t *peerinfo = NULL; + glusterd_peerctx_args_t args = {0}; + int port = 0; char remote_hostname[UNIX_PATH_MAX + 1] = {0,}; GF_ASSERT (req); @@ -2199,6 +2224,10 @@ glusterd_handle_probe_query (rpcsvc_request_t *req) conf = this->private; uuid_unparse (probe_req.uuid, str); + if (probe_req.port) + port = probe_req.port; + else + port = GF_DEFAULT_BASE_PORT; gf_log ("glusterd", GF_LOG_NORMAL, "Received probe from uuid: %s", str); @@ -2206,13 +2235,25 @@ glusterd_handle_probe_query (rpcsvc_request_t *req) ret = glusterd_remote_hostname_get (req, remote_hostname, sizeof (remote_hostname)); if (ret) { - GF_ASSERT (0); + gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname"); goto out; } ret = glusterd_friend_find (probe_req.uuid, remote_hostname, &peerinfo); if ((ret != 0 ) && (!list_empty (&conf->peers))) { rsp.op_ret = -1; rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER; + } else if (ret) { + gf_log ("glusterd", GF_LOG_NORMAL, "Unable to find peerinfo" + " for host: %s (%d)", remote_hostname, port); + args.mode = GD_MODE_SWITCH_ON; + ret = glusterd_friend_add (remote_hostname, port, + GD_FRIEND_STATE_DEFAULT, + NULL, NULL, &peerinfo, 0, &args); + if (ret) { + gf_log ("", GF_LOG_ERROR, "Failed to add peer %s", + remote_hostname); + rsp.op_errno = GF_PROBE_ADD_FAILED; + } } uuid_copy (rsp.uuid, conf->uuid); @@ -2419,7 +2460,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port) NULL, NULL, &peerinfo, 0, &args); } - if (!peerinfo->connected) { + if ((!ret) && (!peerinfo->connected)) { return GLUSTERD_CONNECTION_AWAITED; } diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c index 6061d0276f2..385e62a40df 100644 --- a/xlators/mgmt/glusterd/src/glusterd-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-sm.c @@ -765,7 +765,12 @@ glusterd_friend_sm () event_type = event->event; peerinfo = event->peerinfo; if (!peerinfo) { - GF_ASSERT (0); + gf_log ("glusterd", GF_LOG_CRITICAL, "Received" + " event %s with empty peer info", + glusterd_friend_sm_event_name_get(event_type)); + + GF_FREE (event); + continue; } state = glusterd_friend_state_table[peerinfo->state.state]; diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c index fe9f4e528a0..b8abebf06ca 100644 --- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c +++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c @@ -188,7 +188,9 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov, ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo); if (ret) { - GF_ASSERT (0); + gf_log ("", GF_LOG_ERROR, "received friend add response from" + " unknown peer uuid: %s", str); + goto out; } if (op_ret) @@ -220,6 +222,7 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov, if (ret) goto out; +out: ctx = ((call_frame_t *)myframe)->local; ((call_frame_t *)myframe)->local = NULL; @@ -234,7 +237,6 @@ glusterd3_1_friend_add_cbk (struct rpc_req * req, struct iovec *iov, } if (ctx) glusterd_destroy_probe_ctx (ctx); -out: if (rsp.hostname) free (rsp.hostname);//malloced by xdr GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe)); @@ -285,9 +287,6 @@ glusterd3_1_friend_remove_cbk (struct rpc_req * req, struct iovec *iov, "Received %s from uuid: %s, host: %s, port: %d", (op_ret)?"RJT":"ACC", str, rsp.hostname, rsp.port); - if (op_ret) - goto respond; - inject: ret = glusterd_friend_find (rsp.uuid, ctx->hostname, &peerinfo); @@ -644,6 +643,14 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov, out: if (rsp.op_errstr && strcmp (rsp.op_errstr, "error")) free (rsp.op_errstr); //malloced by xdr + if (dict) { + if (!dict->extra_stdfree && rsp.dict.dict_val) + free (rsp.dict.dict_val); //malloced by xdr + dict_unref (dict); + } else { + if (rsp.dict.dict_val) + free (rsp.dict.dict_val); //malloced by xdr + } GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe)); return ret; } -- cgit