fuse-bridge: Handle graph-switch.

The purpose of this patch is to let protocol/client know when its transports can
be disconnected, without application running on gluster mount noticing any
effects of graph switch.

In order to do this, we migrate all fds and blocked locks to new graph.
Once this migration is complete and there are no in-transit frames as viewed
by fuse-bridge, we send a PARENT_DOWN event to its children. protocol/client
on receiving this event, can disconnect up its transports.

Change-Id: Idcea4bc43e23fb077ac16538b61335ebad84ba16
BUG: 767862
Reviewed-on: http://review.gluster.com/2734
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

5 files changed, 522 insertions, 75 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index e44aca1d0c1..db2deaca29c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -205,11 +205,11 @@ fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        struct fuse_entry_out    feo = {0, };
-        fuse_private_t          *priv = NULL;
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t          *state        = NULL;
+        fuse_in_header_t      *finh         = NULL;
+        struct fuse_entry_out  feo          = {0, };
+        fuse_private_t        *priv         = NULL;
+        inode_t               *linked_inode = NULL;
 
         priv = this->private;
         state = frame->root->state;
@@ -630,11 +630,11 @@ static int
 fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
              int32_t op_ret, int32_t op_errno, fd_t *fd)
 {
-        fuse_state_t         *state = NULL;
-        fuse_in_header_t     *finh  = NULL;
-        fuse_private_t       *priv  = NULL;
-        int32_t               ret   = 0;
-        struct fuse_open_out  foo   = {0, };
+        fuse_state_t         *state  = NULL;
+        fuse_in_header_t     *finh   = NULL;
+        fuse_private_t       *priv   = NULL;
+        int32_t               ret    = 0;
+        struct fuse_open_out  foo    = {0, };
 
         priv = this->private;
         state = frame->root->state;
@@ -677,12 +677,10 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto err;
                 }
 
-                fd_ref (fd);
-
                 if (send_fuse_obj (this, finh, &foo) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "open(%s) got EINTR", state->loc.path);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -695,6 +693,7 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         strerror (op_errno));
 
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -1141,6 +1140,11 @@ fuse_mknod_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1236,6 +1240,11 @@ fuse_mkdir_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1399,6 +1408,11 @@ fuse_symlink_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
                 gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
                 inode_unref (state->loc.inode);
@@ -1516,6 +1530,9 @@ fuse_rename_resume (fuse_state_t *state)
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RENAME `%s (%s)' -> `%s (%s)'",
                 state->finh->unique, state->loc.path, loc_uuid,
@@ -1558,6 +1575,9 @@ fuse_link_resume (fuse_state_t *state)
                 return;
         }
 
+        state->resolve.op_ret = 0;
+        state->resolve2.op_ret = 0;
+
 	if (state->loc.inode) {
 		inode_unref (state->loc.inode);
 		state->loc.inode = NULL;
@@ -1599,14 +1619,14 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  fd_t *fd, inode_t *inode, struct iatt *buf,
                  struct iatt *preparent, struct iatt *postparent)
 {
-        fuse_state_t            *state = NULL;
-        fuse_in_header_t        *finh = NULL;
-        fuse_private_t          *priv = NULL;
-        struct fuse_out_header   fouh = {0, };
-        struct fuse_entry_out    feo = {0, };
-        struct fuse_open_out     foo = {0, };
-        struct iovec             iov_out[3];
-        inode_t                 *linked_inode = NULL;
+        fuse_state_t           *state        = NULL;
+        fuse_in_header_t       *finh         = NULL;
+        fuse_private_t         *priv         = NULL;
+        struct fuse_out_header  fouh         = {0, };
+        struct fuse_entry_out   feo          = {0, };
+        struct fuse_open_out    foo          = {0, };
+        struct iovec            iov_out[3];
+        inode_t                *linked_inode = NULL;
 
         state    = frame->root->state;
         priv     = this->private;
@@ -1645,8 +1665,6 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 inode_unref (linked_inode);
 
-                fd_ref (fd);
-
                 feo.nodeid = inode_to_fuse_nodeid (linked_inode);
 
                 feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
@@ -1671,7 +1689,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "create(%s) got EINTR", state->loc.path);
                         inode_forget (inode, 1);
-                        fd_unref (fd);
+                        gf_fd_put (priv->fdtable, state->fd_no);
                         goto out;
                 }
 
@@ -1681,6 +1699,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         "%"PRIu64": %s => -1 (%s)", finh->unique,
                         state->loc.path, strerror (op_errno));
                 send_fuse_err (this, finh, op_errno);
+                gf_fd_put (priv->fdtable, state->fd_no);
         }
 out:
         free_fuse_state (state);
@@ -1693,7 +1712,8 @@ out:
 void
 fuse_create_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
 
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -1705,15 +1725,26 @@ fuse_create_resume (fuse_state_t *state)
                 return;
         }
 
+        if (state->resolve.op_errno == ENOENT) {
+                state->resolve.op_ret = 0;
+                state->resolve.op_errno = 0;
+        }
+
         if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
+                gf_log (state->this->name, GF_LOG_DEBUG,
+                        "inode already present");
                 inode_unref (state->loc.inode);
         }
 
         state->loc.inode = inode_new (state->loc.parent->table);
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -1799,7 +1830,8 @@ fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_open_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
@@ -1820,7 +1852,10 @@ fuse_open_resume (fuse_state_t *state)
                 return;
         }
 
-        state->fd = fd;
+        priv = state->this->private;
+
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+        state->fd = fd_ref (fd);
         fd->flags = state->flags;
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
@@ -2084,18 +2119,21 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        fd_t                   *fd         = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri    = msg;
+        fd_t                   *new_fd = NULL;
+        fd_t                   *fd     = NULL;
+        uint64_t                val    = 0;
+        int                     ret    = 0;
+        fuse_state_t           *state  = NULL;
+        fuse_fd_ctx_t          *fdctx  = NULL;
+        fuse_private_t         *priv   = NULL;
 
         GET_STATE (this, finh, state);
         fd = FH_TO_FD (fri->fh);
         state->fd = fd;
 
+        priv = this->private;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASE %p", finh->unique, state->fd);
 
@@ -2113,6 +2151,10 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
         fd_unref (fd);
 
+        state->fd = NULL;
+
+        gf_fdptr_put (priv->fdtable, fd);
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
@@ -2156,7 +2198,10 @@ fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_opendir_resume (fuse_state_t *state)
 {
-        fd_t *fd = NULL;
+        fd_t           *fd   = NULL;
+        fuse_private_t *priv = NULL;
+
+        priv = state->this->private;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -2168,7 +2213,8 @@ fuse_opendir_resume (fuse_state_t *state)
         }
 
         fd = fd_create (state->loc.inode, state->finh->pid);
-        state->fd = fd;
+        state->fd = fd_ref (fd);
+        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
 
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": OPENDIR %s", state->finh->unique,
@@ -2331,16 +2377,19 @@ fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri        = msg;
-        fd_t                   *new_fd     = NULL;
-        uint64_t                val        = 0;
-        int                     ret        = 0;
-        fuse_state_t           *state      = NULL;
-        fuse_fd_ctx_t          *fdctx      = NULL;
+        struct fuse_release_in *fri    = msg;
+        fd_t                   *new_fd = NULL;
+        uint64_t                val    = 0;
+        int                     ret    = 0;
+        fuse_state_t           *state  = NULL;
+        fuse_fd_ctx_t          *fdctx  = NULL;
+        fuse_private_t         *priv   = NULL;
 
         GET_STATE (this, finh, state);
         state->fd = FH_TO_FD (fri->fh);
 
+        priv = this->private;
+
         gf_log ("glusterfs-fuse", GF_LOG_TRACE,
                 "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
 
@@ -2360,6 +2409,10 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 
         fd_unref (state->fd);
 
+        gf_fdptr_put (priv->fdtable, state->fd);
+
+        state->fd = NULL;
+
         send_fuse_err (this, finh, 0);
 
         free_fuse_state (state);
@@ -3409,11 +3462,321 @@ fuse_first_lookup (xlator_t *this)
 
 
 int
+fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
+{
+        int          ret          = -1;
+        dict_t      *xattr_req    = NULL;
+        struct iatt  iatt         = {0, };
+
+        if ((loc == NULL) || (xl == NULL)) {
+                goto out;
+        }
+
+        if (loc->inode == NULL) {
+                loc->inode = inode_new (xl->itable);
+                if (loc->inode == NULL) {
+                        goto out;
+                }
+        }
+
+        uuid_copy (loc->gfid, gfid);
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto out;
+        }
+
+        ret = syncop_lookup (xl, loc, xattr_req, &iatt, NULL, NULL);
+        if (ret < 0) {
+                goto out;
+        }
+
+        inode_link (loc->inode, NULL, NULL, &iatt);
+
+        ret = 0;
+out:
+        if (xattr_req != NULL) {
+                dict_unref (xattr_req);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int      ret                = -1;
+        loc_t    loc                = {0, };
+        char     create_in_progress = 0;
+        inode_t *old_inode          = NULL;
+
+        /* could've used pthread_cond_wait, but that requires a cond variable to
+         * be mainted for each fd and that is a bit too much overhead.
+         */
+        do {
+                LOCK (&fd->inode->lock);
+                {
+                        if (uuid_is_null (fd->inode->gfid)) {
+                                create_in_progress = 1;
+                        } else {
+                                create_in_progress = 0;
+                        }
+                }
+                UNLOCK (&fd->inode->lock);
+
+                if (create_in_progress) {
+                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                                "create call on fd (%p) is in progress, "
+                                "hence waiting", fd);
+                        sleep (1);
+                }
+
+        } while (create_in_progress);
+
+        if (fd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, fd);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s)", strerror (errno));
+                }
+        } else {
+                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fd (%p) was not "
+                        "migrated during previous graph switch", fd);
+        }
+
+        loc.path = "";
+        loc.name = NULL;
+
+        ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "name-less lookup of gfid (%s) failed (%s)",
+                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                goto out;
+        }
+
+        old_inode = fd->inode;
+
+        inode_ref (loc.inode);
+
+        LOCK (&fd->inode->lock);
+        {
+                list_del_init (&fd->inode_list);
+        }
+        UNLOCK (&fd->inode->lock);
+
+        LOCK (&fd->lock);
+        {
+                fd->inode = loc.inode;
+        }
+        UNLOCK (&fd->lock);
+
+        if (IA_ISDIR (fd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, fd);
+        } else {
+                ret = syncop_open (new_subvol, &loc, fd->flags, fd);
+        }
+
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "open on gfid (%s) failed (%s)",
+                        uuid_utoa (fd->inode->gfid), strerror (errno));
+                goto out;
+        }
+
+        fd_bind (fd);
+
+        ret = 0;
+out:
+        if (loc.inode != NULL) {
+                inode_unref (loc.inode);
+        }
+
+        if (old_inode != NULL) {
+                inode_unref (old_inode);
+        }
+
+        return ret;
+}
+
+
+int
+fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
+                        xlator_t *new_subvol)
+{
+        fuse_private_t *priv      = NULL;
+        fdentry_t      *fdentries = NULL;
+        uint32_t        count     = 0;
+        fdtable_t      *fdtable   = NULL;
+        int             i         = 0;
+        fd_t           *fd        = NULL;
+
+        priv = this->private;
+
+        fdtable = priv->fdtable;
+
+        fdentries = gf_fd_fdtable_copy_all_fds (fdtable, &count);
+        if (fdentries != NULL) {
+                for (i = 0; i < count; i++) {
+                        fd = fdentries[i].fd;
+                        if (fd != NULL) {
+                                fuse_migrate_fd (this, fd, old_subvol,
+                                                 new_subvol);
+                        }
+                }
+
+                GF_FREE (fdentries);
+        }
+
+        return 0;
+}
+
+
+static int
+fuse_handle_blocked_locks (xlator_t *this, xlator_t *old_subvol,
+                           xlator_t *new_subvol)
+{
+        return 0;
+}
+
+
+static int
+fuse_graph_switch_task (void *data)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = data;
+        if (args == NULL) {
+                goto out;
+        }
+
+        /* don't change the order of handling open fds and blocked locks, since
+         * the act of opening files also reacquires granted locks in new graph.
+         */
+        fuse_handle_opened_fds (args->this, args->old_subvol, args->new_subvol);
+
+        fuse_handle_blocked_locks (args->this, args->old_subvol,
+                                   args->new_subvol);
+
+        pthread_mutex_lock (&args->lock);
+        {
+                args->complete = 1;
+                pthread_cond_broadcast (&args->cond);
+        }
+        pthread_mutex_unlock (&args->lock);
+out:
+        return 0;
+}
+
+
+fuse_graph_switch_args_t *
+fuse_graph_switch_args_alloc (void)
+{
+        fuse_graph_switch_args_t *args = NULL;
+
+        args = GF_CALLOC (1, sizeof (*args), gf_fuse_mt_graph_switch_args_t);
+        if (args == NULL) {
+                goto out;
+        }
+
+        pthread_cond_init (&args->cond, NULL);
+        pthread_mutex_init (&args->lock, NULL);
+
+out:
+        return args;
+}
+
+
+void
+fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
+{
+        if (args == NULL) {
+                goto out;
+        }
+
+        pthread_cond_destroy (&args->cond);
+        pthread_mutex_destroy (&args->lock);
+
+        GF_FREE (args);
+out:
+        return;
+}
+
+
+static int
+fuse_graph_switch_complete (int ret, call_frame_t *frame, void *data)
+{
+        return 0;
+}
+
+
+int
+fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
+                          xlator_t *new_subvol)
+{
+        call_frame_t             *frame = NULL;
+        int32_t                   ret   = -1;
+        fuse_graph_switch_args_t *args  = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                goto out;
+        }
+
+        args = fuse_graph_switch_args_alloc ();
+        if (args == NULL) {
+                goto out;
+        }
+
+        args->this = this;
+        args->old_subvol = old_subvol;
+        args->new_subvol = new_subvol;
+
+        ret = synctask_new (this->ctx->env, fuse_graph_switch_task,
+                            fuse_graph_switch_complete, frame, args);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
+                        "handle graph switch failed");
+                goto out;
+        }
+
+        pthread_mutex_lock (&args->lock);
+        {
+                while (!args->complete) {
+                        ret = pthread_cond_wait (&args->cond, &args->lock);
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "cond_wait failed ret:%d errno:%d", ret,
+                                        errno);
+                        }
+                }
+        }
+        pthread_mutex_unlock (&args->lock);
+
+        ret = 0;
+out:
+        if (args != NULL) {
+                fuse_graph_switch_args_destroy (args);
+        }
+
+        if (frame != NULL) {
+                STACK_DESTROY (frame->root);
+        }
+
+        return ret;
+}
+
+
+int
 fuse_graph_sync (xlator_t *this)
 {
-        fuse_private_t   *priv = NULL;
-        int               need_first_lookup = 0;
-        int               ret = 0;
+        fuse_private_t *priv                = NULL;
+        int             need_first_lookup   = 0;
+        int             ret                 = 0;
+        xlator_t       *old_subvol          = NULL, *new_subvol = NULL;
+        uint64_t        winds_on_old_subvol = 0;
 
         priv = this->private;
 
@@ -3422,7 +3785,8 @@ fuse_graph_sync (xlator_t *this)
                 if (!priv->next_graph)
                         goto unlock;
 
-                priv->active_subvol = priv->next_graph->top;
+                old_subvol = priv->active_subvol;
+                new_subvol = priv->active_subvol = priv->next_graph->top;
                 priv->next_graph = NULL;
                 need_first_lookup = 1;
 
@@ -3444,6 +3808,22 @@ unlock:
                 fuse_first_lookup (this);
         }
 
+        if ((old_subvol != NULL) && (new_subvol != NULL)) {
+                fuse_handle_graph_switch (this, old_subvol, new_subvol);
+
+                pthread_mutex_lock (&priv->sync_mutex);
+                {
+                        old_subvol->switched = 1;
+                        winds_on_old_subvol = old_subvol->winds;
+                }
+                pthread_mutex_unlock (&priv->sync_mutex);
+
+                if (winds_on_old_subvol == 0) {
+                        xlator_notify (old_subvol, GF_EVENT_PARENT_DOWN,
+                                       old_subvol, NULL);
+                }
+        }
+
         return 0;
 }
 
@@ -4068,6 +4448,12 @@ init (xlator_t *this_xl)
         if (!fsname)
                 fsname = "glusterfs";
 
+        priv->fdtable = gf_fd_fdtable_alloc ();
+        if (priv->fdtable == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+                goto cleanup_exit;
+        }
+
         gf_asprintf (&mnt_args, "%s%sallow_other,max_read=131072",
                      priv->read_only ? "ro," : "",
                      priv->acl ? "" : "default_permissions,");
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index d9783f2e054..13c026bcff7 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -54,6 +54,7 @@
 
 #include "list.h"
 #include "dict.h"
+#include "syncop.h"
 
 #if defined(GF_LINUX_HOST_OS) || defined(__NetBSD__)
 #define FUSE_OP_HIGH (FUSE_POLL + 1)
@@ -109,6 +110,7 @@ struct fuse_private {
         unsigned             uid_map_root;
         gf_boolean_t         acl;
         gf_boolean_t         read_only;
+        fdtable_t           *fdtable;
 
         /* For fuse-reverse-validation */
         int                  revchan_in;
@@ -117,6 +119,16 @@ struct fuse_private {
 };
 typedef struct fuse_private fuse_private_t;
 
+struct fuse_graph_switch_args {
+        xlator_t        *this;
+        xlator_t        *old_subvol;
+        xlator_t        *new_subvol;
+        pthread_cond_t   cond;
+        pthread_mutex_t  lock;
+        char             complete;
+};
+typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
+
 #define INVAL_BUF_SIZE (sizeof (struct fuse_out_header) +               \
                         max (sizeof (struct fuse_notify_inval_inode_out), \
                              sizeof (struct fuse_notify_inval_entry_out) + \
@@ -131,6 +143,7 @@ typedef struct fuse_private fuse_private_t;
         do {                                                            \
                 call_frame_t *frame = NULL;                             \
                 xlator_t *xl = NULL;                                    \
+                int32_t   op_ret = 0, op_errno = 0;                     \
                                                                         \
                 frame = get_call_frame_for_req (state);                 \
                 if (!frame) {                                           \
@@ -140,7 +153,7 @@ typedef struct fuse_private fuse_private_t;
                           * better than trying to go on with a NULL     \
                           * frame ...                                   \
                           */                                            \
-                        gf_log ("glusterfs-fuse",                       \
+                        gf_log_callingfn ("glusterfs-fuse",             \
                                 GF_LOG_ERROR,                           \
                                 "FUSE message"                          \
                                 " unique %"PRIu64" opcode %d:"          \
@@ -159,9 +172,27 @@ typedef struct fuse_private fuse_private_t;
                                                                         \
                 xl = state->active_subvol;				\
                 if (!xl) {                                              \
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,         \
-                                "xl is NULL");                          \
-                        send_fuse_err (state->this, state->finh, ENOENT); \
+                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_ERROR, \
+                                          "xl is NULL");                  \
+                        op_errno = ENOENT;                              \
+                        op_ret = -1;                                    \
+                } else if (state->resolve.op_ret < 0) {                 \
+                        op_errno = state->resolve.op_errno;               \
+                        op_ret = -1;                                      \
+/*                        gf_log_callingfn ("glusterfs-fuse", GF_LOG_WARNING, \
+                                          "resolve failed (%s)",             \
+                                          strerror (op_errno));             */ \
+                } else if (state->resolve2.op_ret < 0) {                     \
+                        op_errno = state->resolve2.op_errno;                 \
+                        op_ret = -1;                                         \
+                        /* gf_log_callingfn ("glusterfs-fuse", GF_LOG_WARNING, \
+                                          "resolve of second entity "        \
+                                          "failed (%s)",                     \
+                                          strerror (op_errno));             */ \
+                }                                                            \
+                                                                             \
+                if (op_ret < 0) {                                            \
+                        send_fuse_err (state->this, state->finh, op_errno);  \
                         free_fuse_state (state);                        \
                         STACK_DESTROY (frame->root);                    \
                 } else {                                                \
@@ -293,6 +324,7 @@ typedef struct {
 
         uuid_t         gfid;
         uint32_t       io_flags;
+        int32_t        fd_no;
 } fuse_state_t;
 
 typedef struct fuse_fd_ctx {
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index c794813cf3c..6aee697158c 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -57,6 +57,15 @@ fuse_resolve_wipe (fuse_resolve_t *resolve)
 void
 free_fuse_state (fuse_state_t *state)
 {
+        xlator_t       *this     = NULL;
+        fuse_private_t *priv     = NULL;
+        uint64_t        winds    = 0;
+        char            switched = 0;
+
+        this = state->this;
+
+        priv = this->private;
+
         loc_wipe (&state->loc);
 
         loc_wipe (&state->loc2);
@@ -81,6 +90,18 @@ free_fuse_state (fuse_state_t *state)
         fuse_resolve_wipe (&state->resolve);
         fuse_resolve_wipe (&state->resolve2);
 
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                winds = --state->active_subvol->winds;
+                switched = state->active_subvol->switched;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
+        if ((winds == 0) && (switched)) {
+                xlator_notify (state->active_subvol, GF_EVENT_PARENT_DOWN,
+                               state->active_subvol, NULL);
+        }
+
 #ifdef DEBUG
         memset (state, 0x90, sizeof (*state));
 #endif
@@ -92,8 +113,9 @@ free_fuse_state (fuse_state_t *state)
 fuse_state_t *
 get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
 {
-        fuse_state_t *state = NULL;
-	xlator_t     *active_subvol = NULL;
+        fuse_state_t   *state         = NULL;
+	xlator_t       *active_subvol = NULL;
+        fuse_private_t *priv          = NULL;
 
         state = (void *)GF_CALLOC (1, sizeof (*state),
                                    gf_fuse_mt_fuse_state_t);
@@ -101,7 +123,15 @@ get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
                 return NULL;
 
 	state->this = THIS;
-        active_subvol = fuse_active_subvol (state->this);
+        priv = this->private;
+
+        pthread_mutex_lock (&priv->sync_mutex);
+        {
+                active_subvol = fuse_active_subvol (state->this);
+                active_subvol->winds++;
+        }
+        pthread_mutex_unlock (&priv->sync_mutex);
+
 	state->active_subvol = active_subvol;
 	state->itable = active_subvol->itable;
 
diff --git a/xlators/mount/fuse/src/fuse-mem-types.h b/xlators/mount/fuse/src/fuse-mem-types.h
index 1fb959c3a44..9c6a1c67a7a 100644
--- a/xlators/mount/fuse/src/fuse-mem-types.h
+++ b/xlators/mount/fuse/src/fuse-mem-types.h
@@ -30,6 +30,7 @@ enum gf_fuse_mem_types_ {
         gf_fuse_mt_iov_base,
         gf_fuse_mt_fuse_state_t,
         gf_fuse_mt_fd_ctx_t,
+        gf_fuse_mt_graph_switch_args_t,
         gf_fuse_mt_end
 };
 #endif
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 1af80b93c57..5a09ea0d3e9 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -80,11 +80,13 @@ fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         STACK_DESTROY (frame->root);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
                         "%s/%s: failed to resolve (%s)",
                         uuid_utoa (resolve_loc->pargfid), resolve_loc->name,
                         strerror (op_errno));
+                resolve->op_ret = -1;
+                resolve->op_errno = op_errno;
                 goto out;
         }
 
@@ -141,12 +143,14 @@ fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         STACK_DESTROY (frame->root);
 
         if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
+                gf_log (this->name, (op_errno == ENOENT)
+                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
                         "%s: failed to resolve (%s)",
                         uuid_utoa (resolve->resolve_loc.gfid),
 			strerror (op_errno));
                 loc_wipe (&resolve->resolve_loc);
+                resolve->op_ret = -1;
+                resolve->op_errno = op_errno;
                 goto out;
         }
 
@@ -163,6 +167,7 @@ fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	}
 
 	loc_now->parent = link_inode;
+        uuid_copy (loc_now->pargfid, link_inode->gfid);
 
 	fuse_resolve_entry (state);
 
@@ -237,13 +242,12 @@ fuse_resolve_parent_simple (fuse_state_t *state)
 
         parent = inode_find (state->itable, resolve->pargfid);
 	if (!parent) {
-		resolve->op_ret   = -1;
-		resolve->op_errno = ENOENT;
 		/* non decisive result - parent missing */
 		return 1;
 	}
 
 	loc->parent = parent;
+        uuid_copy (loc->pargfid, resolve->pargfid);
 
 	inode = inode_grep (state->itable, parent, loc->name);
 	if (inode) {
@@ -261,9 +265,6 @@ int
 fuse_resolve_parent (fuse_state_t *state)
 {
         int    ret = 0;
-        loc_t *loc = NULL;
-
-        loc  = state->loc_now;
 
         ret = fuse_resolve_parent_simple (state);
         if (ret > 0) {
@@ -302,9 +303,6 @@ fuse_resolve_inode_simple (fuse_state_t *state)
         if (inode)
 		goto found;
 
-	resolve->op_ret   = -1;
-	resolve->op_errno = ENOENT;
-
         return 1;
 found:
 	loc->inode = inode;
@@ -316,9 +314,6 @@ int
 fuse_resolve_inode (fuse_state_t *state)
 {
         int                 ret = 0;
-        loc_t              *loc = NULL;
-
-        loc  = state->loc_now;
 
         ret = fuse_resolve_inode_simple (state);
 
@@ -344,7 +339,14 @@ fuse_resolve_fd (fuse_state_t *state)
         fd = resolve->fd;
 	active_subvol = fd->inode->table->xl;
 
-	state->active_subvol = active_subvol;
+        if (state->active_subvol != active_subvol) {
+                resolve->op_ret = -1;
+                resolve->op_errno = EBADF;
+                gf_log ("fuse-resolve", GF_LOG_WARNING, "migration of fd (%p) "
+                        "did not complete, failing fop with EBADF", fd);
+        }
+
+	/* state->active_subvol = active_subvol; */
 
         fuse_resolve_continue (state);
 
@@ -434,10 +436,6 @@ fuse_resolve (fuse_state_t *state)
                 fuse_resolve_inode (state);
 
         } else {
-
-                resolve->op_ret = 0;
-                resolve->op_errno = EINVAL;
-
                 fuse_resolve_all (state);
         }