rpc/clnt: handle PARENT_DOWN event appropriately

Change-Id: I4644e944bad4d240d16de47786b9fa277333dba4
BUG: 767862
Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Amar Tumballi <amarts@redhat.com>
Reviewed-on: http://review.gluster.com/2735
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

9 files changed, 168 insertions, 41 deletions

diff --git a/libglusterfs/src/defaults.c b/libglusterfs/src/defaults.c
index f3a8d2832a8..c1382ae8f01 100644
--- a/libglusterfs/src/defaults.c
+++ b/libglusterfs/src/defaults.c
@@ -1235,6 +1235,7 @@ default_notify (xlator_t *this, int32_t event, void *data, ...)
         switch (event)
         {
         case GF_EVENT_PARENT_UP:
+        case GF_EVENT_PARENT_DOWN:
         {
                 xlator_list_t *list = this->children;
 
diff --git a/rpc/rpc-lib/src/rpc-clnt.c b/rpc/rpc-lib/src/rpc-clnt.c
index ec73631d484..b2b20ea3103 100644
--- a/rpc/rpc-lib/src/rpc-clnt.c
+++ b/rpc/rpc-lib/src/rpc-clnt.c
@@ -554,6 +554,12 @@ rpc_clnt_connection_cleanup (rpc_clnt_connection_t *conn)
                 }
 
                 conn->connected = 0;
+
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (clnt->ctx, conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        conn->ping_started = 0;
+                }
         }
         pthread_mutex_unlock (&conn->lock);
 
@@ -842,12 +848,12 @@ int
 rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
                  rpc_transport_event_t event, void *data, ...)
 {
-        rpc_clnt_connection_t  *conn     = NULL;
-        struct rpc_clnt        *clnt     = NULL;
-        int                     ret      = -1;
-        rpc_request_info_t     *req_info = NULL;
-        rpc_transport_pollin_t *pollin   = NULL;
-        struct timeval          tv       = {0, };
+        rpc_clnt_connection_t  *conn        = NULL;
+        struct rpc_clnt        *clnt        = NULL;
+        int                     ret         = -1;
+        rpc_request_info_t     *req_info    = NULL;
+        rpc_transport_pollin_t *pollin      = NULL;
+        struct timeval          tv          = {0, };
 
         conn = mydata;
         if (conn == NULL) {
@@ -864,7 +870,8 @@ rpc_clnt_notify (rpc_transport_t *trans, void *mydata,
 
                 pthread_mutex_lock (&conn->lock);
                 {
-                        if (conn->reconnect == NULL) {
+                        if (!conn->rpc_clnt->disabled
+                            && (conn->reconnect == NULL)) {
                                 tv.tv_sec = 10;
 
                                 conn->reconnect =
@@ -1412,6 +1419,12 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
                 goto out;
         }
 
+        conn = &rpc->conn;
+
+        if (conn->trans == NULL) {
+                goto out;
+        }
+
         rpcreq = mem_get (rpc->reqpool);
         if (rpcreq == NULL) {
                 goto out;
@@ -1431,8 +1444,6 @@ rpc_clnt_submit (struct rpc_clnt *rpc, rpc_clnt_prog_t *prog,
 
         callid = rpc_clnt_new_callid (rpc);
 
-        conn = &rpc->conn;
-
         rpcreq->prog = prog;
         rpcreq->procnum = procnum;
         rpcreq->conn = conn;
@@ -1554,9 +1565,9 @@ rpc_clnt_destroy (struct rpc_clnt *rpc)
                 return;
 
         if (rpc->conn.trans) {
-                rpc->conn.trans->mydata = NULL;
+                rpc_transport_unregister_notify (rpc->conn.trans);
+                rpc_transport_disconnect (rpc->conn.trans);
                 rpc_transport_unref (rpc->conn.trans);
-                //rpc_transport_destroy (rpc->conn.trans);
         }
 
         rpc_clnt_connection_cleanup (&rpc->conn);
@@ -1595,6 +1606,44 @@ rpc_clnt_unref (struct rpc_clnt *rpc)
 
 
 void
+rpc_clnt_disable (struct rpc_clnt *rpc)
+{
+        rpc_clnt_connection_t *conn = NULL;
+
+        if (!rpc) {
+                goto out;
+        }
+
+        conn = &rpc->conn;
+
+        pthread_mutex_lock (&conn->lock);
+        {
+                rpc->disabled = 1;
+
+                if (conn->timer) {
+                        gf_timer_call_cancel (rpc->ctx, conn->timer);
+                        conn->timer = NULL;
+                }
+
+                conn->connected = 0;
+
+                if (conn->ping_timer) {
+                        gf_timer_call_cancel (rpc->ctx, conn->ping_timer);
+                        conn->ping_timer = NULL;
+                        conn->ping_started = 0;
+                }
+
+        }
+        pthread_mutex_unlock (&conn->lock);
+
+        rpc_transport_disconnect (rpc->conn.trans);
+
+out:
+        return;
+}
+
+
+void
 rpc_clnt_reconfig (struct rpc_clnt *rpc, struct rpc_clnt_config *config)
 {
         if (config->rpc_timeout) {
diff --git a/rpc/rpc-lib/src/rpc-clnt.h b/rpc/rpc-lib/src/rpc-clnt.h
index dcd926da93f..4fce0854604 100644
--- a/rpc/rpc-lib/src/rpc-clnt.h
+++ b/rpc/rpc-lib/src/rpc-clnt.h
@@ -187,6 +187,7 @@ typedef struct rpc_clnt {
         glusterfs_ctx_t       *ctx;
         int                   refcount;
         int                   auth_null;
+        char                  disabled;
 } rpc_clnt_t;
 
 
@@ -242,4 +243,8 @@ int rpcclnt_cbk_program_register (struct rpc_clnt *svc,
 
 int
 rpc_clnt_transport_unix_options_build (dict_t **options, char *filepath);
+
+void
+rpc_clnt_disable (struct rpc_clnt *rpc);
+
 #endif /* !_RPC_CLNT_H */
diff --git a/rpc/rpc-lib/src/rpc-transport.c b/rpc/rpc-lib/src/rpc-transport.c
index b97ba61bf2b..8a3b839c497 100644
--- a/rpc/rpc-lib/src/rpc-transport.c
+++ b/rpc/rpc-lib/src/rpc-transport.c
@@ -520,6 +520,20 @@ out:
         return ret;
 }
 
+
+inline int
+rpc_transport_unregister_notify (rpc_transport_t *trans)
+{
+        GF_VALIDATE_OR_GOTO ("rpc-transport", trans, out);
+
+        trans->notify = NULL;
+        trans->mydata = NULL;
+
+out:
+        return 0;
+}
+
+
 //give negative values to skip setting that value
 //this function asserts if both the values are negative.
 //why call it if you dont set it.
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index f838d4baf2c..16e061a5988 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -276,6 +276,9 @@ int
 rpc_transport_register_notify (rpc_transport_t *trans, rpc_transport_notify_t,
                                void *mydata);
 
+int
+rpc_transport_unregister_notify (rpc_transport_t *trans);
+
 int32_t
 rpc_transport_get_peername (rpc_transport_t *this, char *hostname, int hostlen);
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index db2deaca29c..b5059878845 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -354,12 +354,11 @@ fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
         char         *name      = msg;
         fuse_state_t *state     = NULL;
-        int32_t       ret       = -1;
 
         GET_STATE (this, finh, state);
 
-        ret = fuse_resolve_entry_init (state, &state->resolve,
-				       finh->nodeid, name);
+        (void) fuse_resolve_entry_init (state, &state->resolve,
+                                        finh->nodeid, name);
 
         fuse_resolve_and_resume (state, fuse_lookup_resume);
 }
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 8955e237dee..acbc4829bb4 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -2062,7 +2062,7 @@ out:
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
-        clnt_conf_t *conf  = NULL;
+        clnt_conf_t     *conf  = NULL;
 
         conf = this->private;
         if (!conf)
@@ -2076,8 +2076,22 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         "on transport");
 
                 rpc_clnt_start (conf->rpc);
+                break;
         }
-        break;
+
+        case GF_EVENT_PARENT_DOWN:
+                gf_log (this->name, GF_LOG_INFO,
+                        "current graph is no longer active, destroying "
+                        "rpc_client ");
+
+                pthread_mutex_lock (&conf->lock);
+                {
+                        conf->parent_down = 1;
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                rpc_clnt_disable (conf->rpc);
+                break;
 
         default:
                 gf_log (this->name, GF_LOG_DEBUG,
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 00addf34ceb..f7267b44a67 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -98,6 +98,8 @@ typedef struct clnt_conf {
                                               performing lock healing */
         struct timeval         grace_tv;
         gf_timer_t            *grace_timer;
+
+        char                   parent_down;
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 4d6d57528f3..0bc91527aca 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -31,6 +31,7 @@ int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data);
 void client_start_ping (void *data);
 rpc_clnt_prog_t clnt3_1_fop_prog;
 
+
 int
 client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
                            rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
@@ -1838,8 +1839,9 @@ client3_1_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
 out:
         if (rsp.op_ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+                        "remote operation failed: %s remote_fd = %d",
+                        strerror (gf_error_to_errno (rsp.op_errno)),
+                                  local->cmd);
         }
         STACK_UNWIND_STRICT (readdir, frame, rsp.op_ret,
                              gf_error_to_errno (rsp.op_errno), &entries);
@@ -2343,12 +2345,13 @@ int32_t
 client3_1_releasedir (call_frame_t *frame, xlator_t *this,
                       void *data)
 {
-        clnt_conf_t         *conf = NULL;
-        clnt_fd_ctx_t       *fdctx = NULL;
-        clnt_args_t         *args = NULL;
-        gfs3_releasedir_req  req = {{0,},};
-        int64_t              remote_fd = -1;
-        int                  ret = 0;
+        clnt_conf_t         *conf        = NULL;
+        clnt_fd_ctx_t       *fdctx       = NULL;
+        clnt_args_t         *args        = NULL;
+        gfs3_releasedir_req  req         = {{0,},};
+        int64_t              remote_fd   = -1;
+        int                  ret         = 0;
+        char                 parent_down = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -2376,12 +2379,29 @@ client3_1_releasedir (call_frame_t *frame, xlator_t *this,
         pthread_mutex_unlock (&conf->lock);
 
         if (remote_fd != -1) {
-                req.fd = remote_fd;
-                ret = client_submit_request (this, &req, frame, conf->fops,
-                                             GFS3_OP_RELEASEDIR,
-                                             client3_1_releasedir_cbk,
-                                             NULL, NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_releasedir_req);
+                pthread_mutex_lock (&conf->lock);
+                {
+                        parent_down = conf->parent_down;
+                        if (!parent_down) {
+                                rpc_clnt_ref (conf->rpc);
+                        }
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                if (!parent_down) {
+                        req.fd = remote_fd;
+
+                        ret = client_submit_request (this, &req, frame,
+                                                     conf->fops,
+                                                     GFS3_OP_RELEASEDIR,
+                                                     client3_1_releasedir_cbk,
+                                                     NULL, NULL, 0, NULL, 0,
+                                                     NULL,
+                                                     (xdrproc_t)xdr_gfs3_releasedir_req);
+
+                        rpc_clnt_unref (conf->rpc);
+                }
+
                 inode_unref (fdctx->inode);
                 GF_FREE (fdctx);
         }
@@ -2397,12 +2417,13 @@ int32_t
 client3_1_release (call_frame_t *frame, xlator_t *this,
                    void *data)
 {
-        int64_t           remote_fd = -1;
-        clnt_conf_t      *conf      = NULL;
-        clnt_fd_ctx_t    *fdctx     = NULL;
-        clnt_args_t      *args      = NULL;
-        gfs3_release_req  req       = {{0,},};
-        int               ret       = 0;
+        int64_t           remote_fd   = -1;
+        clnt_conf_t      *conf        = NULL;
+        clnt_fd_ctx_t    *fdctx       = NULL;
+        clnt_args_t      *args        = NULL;
+        gfs3_release_req  req         = {{0,},};
+        int               ret         = 0;
+        char              parent_down = 0;
 
         if (!frame || !this || !data)
                 goto unwind;
@@ -2434,11 +2455,26 @@ client3_1_release (call_frame_t *frame, xlator_t *this,
 
                 delete_granted_locks_fd (fdctx);
 
-                ret = client_submit_request (this, &req, frame, conf->fops,
-                                             GFS3_OP_RELEASE,
-                                             client3_1_release_cbk, NULL, NULL,
-                                             0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_release_req);
+                pthread_mutex_lock (&conf->lock);
+                {
+                        parent_down = conf->parent_down;
+                        if (!parent_down) {
+                                rpc_clnt_ref (conf->rpc);
+                        }
+                }
+                pthread_mutex_unlock (&conf->lock);
+
+                if (!parent_down) {
+                        ret = client_submit_request (this, &req, frame,
+                                                     conf->fops,
+                                                     GFS3_OP_RELEASE,
+                                                     client3_1_release_cbk,
+                                                     NULL, NULL,
+                                                     0, NULL, 0, NULL,
+                                                     (xdrproc_t)xdr_gfs3_release_req);
+                        rpc_clnt_unref (conf->rpc);
+                }
+
                 inode_unref (fdctx->inode);
                 GF_FREE (fdctx);
         }
@@ -3612,6 +3648,7 @@ client3_1_fsync (call_frame_t *frame, xlator_t *this,
         req.data = args->flags;
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
+
         ret = client_submit_request (this, &req, frame, conf->fops,
                                      GFS3_OP_FSYNC, client3_1_fsync_cbk, NULL,
                                      NULL, 0, NULL, 0,
@@ -4916,6 +4953,9 @@ client3_1_readdir (call_frame_t *frame, xlator_t *this,
         req.size = args->size;
         req.offset = args->offset;
         req.fd = remote_fd;
+
+        local->cmd = remote_fd;
+
         memcpy (req.gfid, args->fd->inode->gfid, 16);
 
         ret = client_submit_request (this, &req, frame, conf->fops,