mgmt/glusterd: delay probe till connection happens

Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Signed-off-by: Vijay Bellur <vijay@dev.gluster.com>

BUG: 1607 ()
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1607

10 files changed, 226 insertions, 178 deletions

diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index 6548f73b0..856fcfb50 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -1000,6 +1000,8 @@ rpc_clnt_init (struct rpc_clnt_config *config, dict_t *options,
                 pthread_mutex_destroy (&rpc->lock);
                 GF_FREE (rpc);
                 rpc = NULL;
+                if (options)
+                        dict_unref (options);
                 goto out;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index ef39f76d2..55c01e975 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -182,15 +182,28 @@ glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
         glusterd_peerinfo_t             *peerinfo = NULL;
         glusterd_friend_sm_event_t      *event = NULL;
         glusterd_friend_req_ctx_t       *ctx = NULL;
+        glusterd_peerctx_args_t         args = {0};
+        char                            rhost[UNIX_PATH_MAX + 1] = {0};
+        uuid_t                          friend_uuid = {0};
+        char                            uuid_str[50] = {0,};
 
+        uuid_unparse (uuid, uuid_str);
+        uuid_parse (uuid_str, friend_uuid);
         if (!port)
                 port = 6969; // TODO: use define values.
 
-        ret = glusterd_friend_find (uuid, hostname, &peerinfo);
+        ret = glusterd_remote_hostname_get (req, rhost, sizeof (rhost));
+        ret = glusterd_friend_find (uuid, rhost, &peerinfo);
 
         if (ret) {
                 gf_log ("glusterd", GF_LOG_NORMAL,
-                         "Unable to find peer");
+                        " for host: %s (%d)", rhost, port);
+                args.mode = GD_MODE_SWITCH_ON;
+                args.req  = NULL;
+                ret = glusterd_friend_add (rhost, port,
+                                           GD_FRIEND_STATE_DEFAULT,
+                                           &friend_uuid, NULL, &peerinfo, 0,
+                                           &args);
 
         }
 
@@ -240,6 +253,8 @@ out:
                         GF_FREE (ctx);
         }
 
+        if (0 == peerinfo->connected)
+                ret = GLUSTERD_CONNECTION_AWAITED;
         return ret;
 }
 
@@ -2396,8 +2411,10 @@ glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
                                           dict);
 
 out:
-        if (ret && dict)
-                dict_unref (dict);
+        if (GLUSTERD_CONNECTION_AWAITED != ret) {
+                if (ret && dict)
+                        dict_unref (dict);
+        }
         if (friend_req.hostname)
                 free (friend_req.hostname);//malloced by xdr
 
@@ -2451,6 +2468,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         int                     i = 1;
         int                     count = 0;
         uuid_t                  uuid = {0,};
+        glusterd_peerctx_args_t args = {0};
 
         GF_ASSERT (req);
 
@@ -2490,6 +2508,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
         if (ret)
                 goto out;
 
+        args.mode = GD_MODE_SWITCH_ON;
         while ( i <= count) {
                 snprintf (key, sizeof (key), "friend%d.uuid", i);
                 ret = dict_get_str (dict, key, &uuid_buf);
@@ -2519,7 +2538,7 @@ glusterd_handle_friend_update (rpcsvc_request_t *req)
 
                 ret = glusterd_friend_add (hostname, friend_req.port,
                                            GD_FRIEND_STATE_BEFRIENDED,
-                                           &uuid, NULL, &peerinfo, 0);
+                                           &uuid, NULL, &peerinfo, 0, &args);
 
                 i++;
         }
@@ -2542,7 +2561,6 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
         glusterd_conf_t     *conf = NULL;
         gd1_mgmt_probe_req  probe_req = {{0},};
         gd1_mgmt_probe_rsp  rsp = {{0},};
-        glusterd_peer_hostname_t        *name = NULL;
         glusterd_peerinfo_t             *peerinfo = NULL;
         char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
@@ -2570,16 +2588,7 @@ glusterd_handle_probe_query (rpcsvc_request_t *req)
                 goto out;
         }
         ret = glusterd_friend_find (probe_req.uuid, remote_hostname, &peerinfo);
-        if ((ret == 0 ) || list_empty (&conf->peers)) {
-                ret = glusterd_peer_hostname_new (probe_req.hostname, &name);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to get new peer_hostname");
-                } else {
-                        list_add_tail (&name->hostname_list, &conf->hostnames);
-                }
-
-        } else {
+        if ((ret != 0 ) && (!list_empty (&conf->peers))) {
                 rsp.op_ret = -1;
                 rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
         }
@@ -2623,7 +2632,8 @@ glusterd_friend_add (const char *hoststr, int port,
                      uuid_t *uuid,
                      struct rpc_clnt    *rpc,
                      glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore)
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args)
 {
         int                     ret = 0;
         glusterd_conf_t         *priv = NULL;
@@ -2632,14 +2642,25 @@ glusterd_friend_add (const char *hoststr, int port,
         struct rpc_clnt_config  rpc_cfg = {0,};
         glusterd_peer_hostname_t *name = NULL;
         char                    *hostname = NULL;
+        glusterd_peerctx_t     *peerctx = NULL;
 
         priv = THIS->private;
 
-        peerinfo = GF_CALLOC (1, sizeof(*peerinfo), gf_gld_mt_peerinfo_t);
+        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
+        if (!peerctx) {
+                ret = -1;
+                goto out;
+        }
+        peerinfo = GF_CALLOC (1, sizeof (*peerinfo), gf_gld_mt_peerinfo_t);
 
-        if (!peerinfo)
-                return -1;
+        if (!peerinfo) {
+                ret = -1;
+                goto out;
+        }
 
+        if (args)
+                peerctx->args = *args;
+        peerctx->peerinfo = peerinfo;
         if (friend)
                 *friend = peerinfo;
 
@@ -2650,7 +2671,7 @@ glusterd_friend_add (const char *hoststr, int port,
                 if (ret)
                         goto out;
                 list_add_tail (&peerinfo->hostnames, &name->hostname_list);
-                rpc_cfg.remote_host = gf_strdup (hoststr);
+                rpc_cfg.remote_host = (char *)hoststr;
                 peerinfo->hostname = gf_strdup (hoststr);
         }
         INIT_LIST_HEAD (&peerinfo->uuid_list);
@@ -2664,8 +2685,10 @@ glusterd_friend_add (const char *hoststr, int port,
 
         if (hoststr) {
                 options = dict_new ();
-                if (!options)
-                        return -1;
+                if (!options) {
+                        ret = -1;
+                        goto out;
+                }
 
                 hostname = gf_strdup((char*)hoststr);
                 if (!hostname) {
@@ -2701,7 +2724,7 @@ glusterd_friend_add (const char *hoststr, int port,
                 }
 
                 ret = rpc_clnt_register_notify (rpc, glusterd_rpc_notify,
-                                                peerinfo);
+                                                peerctx);
 
                 peerinfo->rpc = rpc;
 
@@ -2712,9 +2735,21 @@ glusterd_friend_add (const char *hoststr, int port,
 
 
 out:
+        if (ret) {
+                if (peerctx)
+                        GF_FREE (peerctx);
+                if (rpc) {
+                        (void) rpc_clnt_unref (rpc);
+                }
+                if (peerinfo) {
+                        peerinfo->rpc = NULL;
+                        (void) glusterd_friend_cleanup (peerinfo);
+                }
+                if (options)
+                        dict_unref (options);
+        }
+
         gf_log ("glusterd", GF_LOG_NORMAL, "connect returned %d", ret);
-        if (rpc_cfg.remote_host)
-                GF_FREE (rpc_cfg.remote_host);
         return ret;
 }
 
@@ -2725,8 +2760,7 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
 {
         int                             ret = -1;
         glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
+        glusterd_peerctx_args_t         args = {0};
 
         GF_ASSERT (hoststr);
 
@@ -2735,38 +2769,11 @@ glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port)
         if (ret) {
                 gf_log ("glusterd", GF_LOG_NORMAL, "Unable to find peerinfo"
                         " for host: %s (%d)", hoststr, port);
+                args.mode = GD_MODE_ON;
+                args.req  = req;
                 ret = glusterd_friend_add ((char *)hoststr, port,
                                            GD_FRIEND_STATE_DEFAULT,
-                                           NULL, NULL, &peerinfo, 0);
-        }
-
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_PROBE, &event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get new event");
-                return ret;
-        }
-
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
-
-        if (!ctx) {
-                return ret;
-        }
-
-        ctx->hostname = gf_strdup (hoststr);
-        ctx->port = port;
-        ctx->req = req;
-
-        event->peerinfo = peerinfo;
-        event->ctx = ctx;
-
-        ret = glusterd_friend_sm_inject_event (event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                return ret;
+                                           NULL, NULL, &peerinfo, 0, &args);
         }
 
         if (!peerinfo->connected) {
@@ -3420,6 +3427,54 @@ out:
         return ret;
 }
 
+static int
+glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
+{
+        GF_ASSERT (peerctx);
+
+        glusterd_friend_sm_event_t      *event = NULL;
+        glusterd_probe_ctx_t            *ctx = NULL;
+        int                             ret = -1;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+
+
+        ret = glusterd_friend_sm_new_event
+                        (GD_FRIEND_EVENT_CONNECTED, &event);
+
+        if (ret) {
+                gf_log ("", GF_LOG_ERROR, "Unable to get new event");
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!ctx) {
+                ret = -1;
+                gf_log ("", GF_LOG_ERROR, "Memory not available");
+                goto out;
+        }
+
+        peerinfo = peerctx->peerinfo;
+        ctx->hostname = gf_strdup (peerinfo->hostname);
+        ctx->port = peerinfo->port;
+        ctx->req = peerctx->args.req;
+
+        event->peerinfo = peerinfo;
+        event->ctx = ctx;
+
+        ret = glusterd_friend_sm_inject_event (event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject "
+                        "EVENT_CONNECTED ret = %d", ret);
+                goto out;
+        }
+
+out:
+        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
 int
 glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                      void *data)
@@ -3429,8 +3484,10 @@ glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         glusterd_conf_t         *conf = NULL;
         int                     ret = 0;
         glusterd_peerinfo_t     *peerinfo = NULL;
+        glusterd_peerctx_t     *peerctx = NULL;
 
-        peerinfo = mydata;
+        peerctx = mydata;
+        peerinfo = peerctx->peerinfo;
         this = THIS;
         conf = this->private;
 
@@ -3438,11 +3495,8 @@ glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
         switch (event) {
         case RPC_CLNT_CONNECT:
         {
-
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
                 peerinfo->connected = 1;
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
 
                 if ((ret < 0) || (strcasecmp (handshake, "on"))) {
                         //ret = glusterd_handshake (this, peerinfo->rpc);
@@ -3451,6 +3505,16 @@ glusterd_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
                         //conf->rpc->connected = 1;
                         ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
                 }
+
+                if (GD_MODE_ON == peerctx->args.mode) {
+                        ret = glusterd_event_connected_inject (peerctx);
+                        peerctx->args.req = NULL;
+                } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+                        peerctx->args.mode = GD_MODE_ON;
+                }
+
+                glusterd_friend_sm ();
+                glusterd_op_sm ();
                 break;
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index 8b99e9e4c..b779dd469 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -58,7 +58,8 @@ enum gf_gld_mem_types_ {
         gf_gld_mt_log_filename_ctx_t            = gf_common_mt_end + 32,
         gf_gld_mt_log_locate_ctx_t              = gf_common_mt_end + 33,
         gf_gld_mt_log_rotate_ctx_t              = gf_common_mt_end + 34,
-        gf_gld_mt_end                           = gf_common_mt_end + 35
+        gf_gld_mt_peerctx_t                     = gf_common_mt_end + 35,
+        gf_gld_mt_end                           = gf_common_mt_end + 36
 };
 #endif
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index c5e1800cd..f540658c3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -67,8 +67,6 @@ glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
                 dict_unref (ctx->vols);
         if (ctx->hostname)
                 GF_FREE (ctx->hostname);
-        if (ctx->remote_hostname)
-                GF_FREE (ctx->remote_hostname);
         GF_FREE (ctx);
 }
 
@@ -99,14 +97,53 @@ static int
 glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
 {
         int ret = 0;
-        glusterd_friend_update_ctx_t    *ev_ctx = NULL;
+        glusterd_peerinfo_t             *peerinfo = NULL;
+        glusterd_friend_sm_event_t      *new_event = NULL;
+        glusterd_probe_ctx_t            *new_ev_ctx = NULL;
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
-        ev_ctx = ctx;
-        ret = glusterd_probe_begin (NULL, ev_ctx->remote_hostname, 0);
+        peerinfo = event->peerinfo;
+        ret = glusterd_friend_sm_new_event
+                (GD_FRIEND_EVENT_PROBE, &new_event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get new new_event");
+                ret = -1;
+                goto out;
+        }
+
+        new_ev_ctx = GF_CALLOC (1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
+
+        if (!new_ev_ctx) {
+                ret = -1;
+                goto out;
+        }
+
+        new_ev_ctx->hostname = gf_strdup (peerinfo->hostname);
+        new_ev_ctx->port = peerinfo->port;
+        new_ev_ctx->req = NULL;
 
+        new_event->peerinfo = peerinfo;
+        new_event->ctx = new_ev_ctx;
+
+        ret = glusterd_friend_sm_inject_event (new_event);
+
+        if (ret) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject new_event %d, "
+                                "ret = %d", new_event->event, ret);
+        }
+
+out:
+        if (ret) {
+                if (new_event)
+                        GF_FREE (new_event);
+                if (new_ev_ctx->hostname)
+                        GF_FREE (new_ev_ctx->hostname);
+                if (new_ev_ctx)
+                        GF_FREE (new_ev_ctx);
+        }
         gf_log ("", GF_LOG_DEBUG, "returning with %d", ret);
         return ret;
 }
@@ -291,14 +328,9 @@ glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
         peerinfo = event->peerinfo;
         GF_ASSERT (peerinfo);
 
-        uuid_clear (peerinfo->uuid);
-
         ret = glusterd_xfer_friend_remove_resp (ev_ctx->req, ev_ctx->hostname,
                                                 ev_ctx->port);
 
-        peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
-        peerinfo->rpc = NULL;
-
         ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
                                             &new_event);
 
@@ -353,7 +385,6 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         int                             status = 0;
         int32_t                         op_ret = -1;
         int32_t                         op_errno = 0;
-        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
 
         GF_ASSERT (ctx);
         ev_ctx = ctx;
@@ -395,14 +426,6 @@ glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
         uuid_copy (new_ev_ctx->uuid, ev_ctx->uuid);
         new_ev_ctx->hostname = gf_strdup (ev_ctx->hostname);
 
-        ret = glusterd_remote_hostname_get (ev_ctx->req, remote_hostname,
-                                            sizeof (remote_hostname));
-        if (ret) {
-                ret = -1;
-                goto out;
-        }
-
-        new_ev_ctx->remote_hostname = gf_strdup (remote_hostname);
         new_event->ctx = new_ev_ctx;
 
         glusterd_friend_sm_inject_event (new_event);
@@ -446,6 +469,7 @@ glusterd_sm_t glusterd_state_default [] = {
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe}, //EVENT_CONNECTED
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
 };
 
@@ -462,6 +486,7 @@ glusterd_sm_t  glusterd_state_req_sent [] = {
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_CONNECTED
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -477,6 +502,7 @@ glusterd_sm_t  glusterd_state_req_rcvd [] = {
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_CONNECTED
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -486,12 +512,13 @@ glusterd_sm_t  glusterd_state_befriended [] = {
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},//EVENT_CONNECTED
         {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -507,6 +534,7 @@ glusterd_sm_t  glusterd_state_req_sent_rcvd [] = {
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_CONNECTED
         {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -514,14 +542,15 @@ glusterd_sm_t  glusterd_state_rejected [] = {
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_NONE,
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe}, //EVENT_PROBE,
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
+        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
         {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},//EVENT_CONNECTED
         {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -537,6 +566,7 @@ glusterd_sm_t  glusterd_state_req_accepted [] = {
         {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},//EVENT_CONNECTED
         {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -552,6 +582,7 @@ glusterd_sm_t  glusterd_state_unfriend_sent [] = {
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_REMOVE_FRIEND
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
         {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
+        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_CONNECTED
         {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_MAX
 };
 
@@ -622,60 +653,6 @@ glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
 }
 
 int
-glusterd_check_and_add_friend (glusterd_friend_sm_event_t *event)
-{
-        rpcsvc_request_t                *req = NULL;
-        glusterd_peerinfo_t             *peerinfo   = NULL;
-        glusterd_friend_sm_event_type_t  event_type = 0;
-        glusterd_friend_req_ctx_t       *fr_ctx = NULL;
-        glusterd_probe_ctx_t            *pb_ctx = NULL;
-        gf_boolean_t                    add_friend = _gf_false;
-        char                            rhost[UNIX_PATH_MAX + 1] = {0};
-        char                            *host_str = NULL;
-        int                             port       = 6969; //TODO, use standard
-        int                             ret = 0;
-
-        peerinfo = event->peerinfo;
-        event_type = event->event;
-
-        if (!peerinfo &&
-           (GD_FRIEND_EVENT_PROBE == event_type)) {
-                add_friend = _gf_true;
-                pb_ctx = event->ctx;
-                req = pb_ctx->req;
-        }
-        if (!peerinfo &&
-            (GD_FRIEND_EVENT_RCVD_FRIEND_REQ == event_type)) {
-                add_friend = _gf_true;
-                fr_ctx = event->ctx;
-                req = fr_ctx->req;
-        }
-        if (add_friend) {
-                if (req) {
-                        ret = glusterd_remote_hostname_get (req, rhost,
-                                                            sizeof (rhost));
-                        if (!ret)
-                                host_str = rhost;
-                }
-                ret = glusterd_friend_add ((const char*)host_str, port,
-                                          GD_FRIEND_STATE_DEFAULT,
-                                          NULL, NULL, &peerinfo, 0);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to add peer, "
-                                "ret = %d", ret);
-                        ret = 1;
-                        goto out;
-                }
-                GF_ASSERT (peerinfo);
-                event->peerinfo = peerinfo;
-        }
-        ret = 0;
-
-out:
-        return ret;
-}
-
-int
 glusterd_friend_sm ()
 {
         glusterd_friend_sm_event_t      *event      = NULL;
@@ -686,22 +663,16 @@ glusterd_friend_sm ()
         glusterd_peerinfo_t             *peerinfo   = NULL;
         glusterd_friend_sm_event_type_t  event_type = 0;
         gf_boolean_t                     is_await_conn = _gf_false;
-        int                              loop       = 0;
 
         while (!list_empty (&gd_friend_sm_queue)) {
                 list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue, list) {
 
                         list_del_init (&event->list);
                         event_type = event->event;
-
-
-                        loop = glusterd_check_and_add_friend (event);
-                        if (loop)
-                                continue;
-
                         peerinfo = event->peerinfo;
-                        if (!peerinfo)
-                                goto out;
+                        if (!peerinfo) {
+                                GF_ASSERT (0);
+                        }
 
                         state = glusterd_friend_state_table[peerinfo->state.state];
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index 1b845b902..0ad89465b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -84,7 +84,21 @@ struct glusterd_peerinfo_ {
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
 
+typedef enum glusterd_ev_gen_mode_ {
+        GD_MODE_OFF,
+        GD_MODE_ON,
+        GD_MODE_SWITCH_ON
+} glusterd_ev_gen_mode_t;
 
+typedef struct glusterd_peer_ctx_args_ {
+        rpcsvc_request_t        *req;
+        glusterd_ev_gen_mode_t  mode;
+} glusterd_peerctx_args_t;
+
+typedef struct glusterd_peer_ctx_ {
+        glusterd_peerctx_args_t        args;
+        glusterd_peerinfo_t            *peerinfo;
+} glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
         GD_FRIEND_EVENT_NONE = 0,
@@ -98,6 +112,7 @@ typedef enum glusterd_friend_sm_event_type_ {
         GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
         GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
         GD_FRIEND_EVENT_REMOVE_FRIEND,
+        GD_FRIEND_EVENT_CONNECTED,
         GD_FRIEND_EVENT_MAX
 } glusterd_friend_sm_event_type_t;
 
@@ -121,7 +136,6 @@ typedef struct glusterd_sm_ {
 typedef struct glusterd_friend_req_ctx_ {
         uuid_t                  uuid;
         char                    *hostname;
-        char                    *remote_hostname;
         rpcsvc_request_t        *req;
         int                      port;
         dict_t                  *vols;
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 498a063a5..b0fc917fd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -1294,6 +1294,7 @@ glusterd_store_retrieve_peers (xlator_t *this)
         glusterd_store_iter_t   *iter = NULL;
         char                    *key = NULL;
         char                    *value = NULL;
+        glusterd_peerctx_args_t args = {0};
 
         GF_ASSERT (this);
         priv = this->private;
@@ -1354,8 +1355,9 @@ glusterd_store_retrieve_peers (xlator_t *this)
 
                 (void) glusterd_store_iter_destroy (iter);
 
+                args.mode = GD_MODE_SWITCH_ON;
                 ret = glusterd_friend_add (hostname, 0, state, &uuid,
-                                           NULL, &peerinfo, 1);
+                                           NULL, &peerinfo, 1, &args);
 
                 if (ret)
                         goto out;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 4d21bbe95..f99ddde9c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -590,7 +590,6 @@ glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
         int32_t                 ret = -1;
         glusterd_peerinfo_t     *peerinfo = NULL;
         glusterd_conf_t         *priv = NULL;
-        glusterd_peer_hostname_t        *host = NULL;
 
         priv = THIS->private;
         GF_ASSERT (priv);
@@ -601,15 +600,6 @@ glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
 
         if (!ret) {
                 uuid_copy (brickinfo->uuid, peerinfo->uuid);
-        } else {
-                list_for_each_entry (host, &priv->hostnames, hostname_list) {
-                        if (!strcmp (host->hostname, brickinfo->hostname)) {
-                                uuid_copy (brickinfo->uuid, priv->uuid);
-                                ret = 0;
-                                break;
-                        }
-
-                }
         }
 
         if (ret) {
@@ -729,7 +719,12 @@ int32_t
 glusterd_friend_cleanup (glusterd_peerinfo_t *peerinfo)
 {
         GF_ASSERT (peerinfo);
+        glusterd_peerctx_t *peerctx = NULL;
+
         if (peerinfo->rpc) {
+                peerctx = peerinfo->rpc->mydata;
+                peerinfo->rpc->mydata = NULL;
+                GF_FREE (peerctx);
                 peerinfo->rpc = rpc_clnt_unref (peerinfo->rpc);
                 peerinfo->rpc = NULL;
         }
@@ -974,6 +969,7 @@ glusterd_peer_destroy (glusterd_peerinfo_t *peerinfo)
                                   hostname_list) {
                 list_del_init (&name->hostname_list);
                 GF_FREE (name->hostname);
+                GF_FREE (name);
         }
 
         list_del_init (&peerinfo->hostnames);
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index b64b001f7..0003fbef5 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -390,7 +390,6 @@ init (xlator_t *this)
         GF_VALIDATE_OR_GOTO(this->name, conf, out);
         INIT_LIST_HEAD (&conf->peers);
         INIT_LIST_HEAD (&conf->volumes);
-        INIT_LIST_HEAD (&conf->hostnames);
         pthread_mutex_init (&conf->mutex, NULL);
         conf->rpc = rpc;
         conf->mgmt = &glusterd3_1_mgmt_prog;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 500c4261c..2eb430f27 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -70,7 +70,6 @@ typedef struct {
         rpc_clnt_prog_t   *mgmt;
         struct pmap_registry *pmap;
         struct list_head  volumes;
-        struct list_head  hostnames;
         struct list_head  xprt_list;
         glusterd_store_handle_t *handle;
         gf_timer_t *timer;
@@ -245,7 +244,8 @@ glusterd_friend_add (const char *hoststr, int port,
                      glusterd_friend_sm_state_t state,
                      uuid_t *uuid, struct rpc_clnt    *rpc,
                      glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore);
+                     gf_boolean_t restore,
+                     glusterd_peerctx_args_t *args);
 
 int
 glusterd_friend_remove (uuid_t uuid, char *hostname);
diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
index fe21e085d..f4b90f6ca 100644
--- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
+++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c
@@ -92,8 +92,9 @@ glusterd3_1_probe_cbk (struct rpc_req *req, struct iovec *iov,
                                                       ctx->hostname, ctx->port);
                 }
 
-                ret = rsp.op_ret;
+                glusterd_destroy_probe_ctx (ctx);
                 (void) glusterd_friend_remove (rsp.uuid, rsp.hostname);
+                ret = rsp.op_ret;
                 goto out;
         }
         ret = glusterd_friend_find (rsp.uuid, rsp.hostname, &peerinfo);
@@ -502,7 +503,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "";
+                rsp.op_errstr = "error";
                 goto out;
         }
 
@@ -511,7 +512,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
                 gf_log ("", GF_LOG_ERROR, "error");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "";
+                rsp.op_errstr = "error";
                 goto out;
         }
         uuid_unparse (rsp.uuid, str);
@@ -550,7 +551,7 @@ glusterd3_1_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
         }
 
 out:
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, ""))
+        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
                 free (rsp.op_errstr); //malloced by xdr
         return ret;
 }
@@ -631,7 +632,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
         if (-1 == req->rpc_status) {
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "";
+                rsp.op_errstr = "error";
                 goto out;
         }
 
@@ -640,7 +641,7 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                 gf_log ("", GF_LOG_ERROR, "error");
                 rsp.op_ret   = -1;
                 rsp.op_errno = EINVAL;
-                rsp.op_errstr = "";
+                rsp.op_errstr = "error";
                 goto out;
         }
         uuid_unparse (rsp.uuid, str);
@@ -657,6 +658,8 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                                 "failed to "
                                 "unserialize rsp-buffer to dictionary");
                         goto out;
+                } else {
+                        dict->extra_stdfree = rsp.dict.dict_val;
                 }
         }
 
@@ -707,13 +710,10 @@ glusterd3_1_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
                 glusterd_op_sm ();
         }
 
-        return ret;
-
-
 out:
         if (dict)
                 dict_unref (dict);
-        if (rsp.op_errstr && strcmp (rsp.op_errstr, ""))
+        if (rsp.op_errstr && strcmp (rsp.op_errstr, "error"))
                 free (rsp.op_errstr); //malloced by xdr
         return ret;
 }
@@ -805,7 +805,7 @@ glusterd3_1_friend_add (call_frame_t *frame, xlator_t *this,
                 goto out;
 
         uuid_copy (req.uuid, priv->uuid);
-        req.hostname = gf_strdup (peerinfo->hostname);
+        req.hostname = peerinfo->hostname;
         req.port = peerinfo->port;
 
         ret = dict_allocate_and_serialize (vols, &req.vols.vols_val,
@@ -826,9 +826,6 @@ out:
         if (vols)
                 dict_unref (vols);
 
-        if (req.hostname)
-                GF_FREE (req.hostname);
-
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }
@@ -1289,6 +1286,8 @@ glusterd_handle_rpc_msg (rpcsvc_request_t *req)
 
                 case GD_MGMT_FRIEND_ADD:
                         ret = glusterd_handle_incoming_friend_req (req);
+                        if (ret == GLUSTERD_CONNECTION_AWAITED)
+                                return 0;
                         break;
 
                 case GD_MGMT_CLUSTER_LOCK: