fuse: create a new fd during fd-migration.

Migration of fd to new graph involves creation of a new fd
to be used only for calls sent in that graph.

Earlier approach of using same fd across all graphs, with the
associated inode always guaranteed to be the one valid in
currently active graph, had issues because of the broken
immutability of the association of fd with an inode
(for the life of fd).

With this patch, there will be a basefd, which the kernel will be
aware of. This basefd, contains a mapping of an fd which is valid
in currently active graph.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Change-Id: I2b459f05bc2690a66498be107fad6444e3158138
BUG: 802414
Reviewed-on: http://review.gluster.org/3566
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

3 files changed, 370 insertions, 130 deletions

diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index 234af8041..e46283793 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -708,7 +708,7 @@ fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
         GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
                                         -EINVAL);
 
-        fdctx = fuse_fd_ctx_check_n_create (this, fd);
+        fdctx = fuse_fd_ctx_get (this, fd);
         if (!fdctx) {
                 ret = -ENOMEM;
                 goto out;
@@ -792,8 +792,9 @@ fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (ret < 0) {
                         op_errno = -ret;
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "cannot inherit direct-io values from fds "
-                                "already opened");
+                                "cannot inherit direct-io values for fd "
+                                "(ptr:%p inode-gfid:%s) from fds already "
+                                "opened", fd, uuid_utoa (fd->inode->gfid));
                         goto err;
                 }
 
@@ -1819,6 +1820,7 @@ fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 #endif
                 iov_out[2].iov_base = &foo;
                 iov_out[2].iov_len = sizeof (foo);
+
                 if (send_fuse_iov (this, finh, iov_out, 3) == ENOENT) {
                         gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
                                 "create(%s) got EINTR", state->loc.path);
@@ -1846,8 +1848,9 @@ out:
 void
 fuse_create_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.parent) {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
@@ -1873,6 +1876,25 @@ fuse_create_resume (fuse_state_t *state)
         state->loc.inode = inode_new (state->loc.parent->table);
 
         fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE cannot create a new fd",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64" CREATE creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
 
         priv = state->this->private;
 
@@ -1969,8 +1991,9 @@ fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_open_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         if (!state->loc.inode) {
                 gf_log ("glusterfs-fuse", GF_LOG_ERROR,
@@ -1991,6 +2014,17 @@ fuse_open_resume (fuse_state_t *state)
                 return;
         }
 
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPEN creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
         priv = state->this->private;
 
         state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
@@ -2259,14 +2293,14 @@ fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri    = msg;
-        fd_t                   *new_fd = NULL;
-        fd_t                   *fd     = NULL;
-        uint64_t                val    = 0;
-        int                     ret    = 0;
-        fuse_state_t           *state  = NULL;
-        fuse_fd_ctx_t          *fdctx  = NULL;
-        fuse_private_t         *priv   = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        fd_t                   *fd        = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         fd = FH_TO_FD (fri->fh);
@@ -2281,9 +2315,9 @@ fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -2338,8 +2372,9 @@ fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
 void
 fuse_opendir_resume (fuse_state_t *state)
 {
-        fd_t           *fd   = NULL;
-        fuse_private_t *priv = NULL;
+        fd_t           *fd    = NULL;
+        fuse_private_t *priv  = NULL;
+        fuse_fd_ctx_t  *fdctx = NULL;
 
         priv = state->this->private;
 
@@ -2353,6 +2388,25 @@ fuse_opendir_resume (fuse_state_t *state)
         }
 
         fd = fd_create (state->loc.inode, state->finh->pid);
+        if (fd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR fd creation failed",
+                        state->finh->unique);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+        }
+
+        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
+        if (fdctx == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "%"PRIu64": OPENDIR creation of fdctx failed",
+                        state->finh->unique);
+                fd_unref (fd);
+                send_fuse_err (state->this, state->finh, ENOMEM);
+                free_fuse_state (state);
+                return;
+        }
+
         state->fd = fd_ref (fd);
         state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
 
@@ -2516,13 +2570,13 @@ fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 static void
 fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
-        struct fuse_release_in *fri    = msg;
-        fd_t                   *new_fd = NULL;
-        uint64_t                val    = 0;
-        int                     ret    = 0;
-        fuse_state_t           *state  = NULL;
-        fuse_fd_ctx_t          *fdctx  = NULL;
-        fuse_private_t         *priv   = NULL;
+        struct fuse_release_in *fri       = msg;
+        fd_t                   *activefd = NULL;
+        uint64_t                val       = 0;
+        int                     ret       = 0;
+        fuse_state_t           *state     = NULL;
+        fuse_fd_ctx_t          *fdctx     = NULL;
+        fuse_private_t         *priv      = NULL;
 
         GET_STATE (this, finh, state);
         state->fd = FH_TO_FD (fri->fh);
@@ -2537,9 +2591,9 @@ fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
         if (!ret) {
                 fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
                 if (fdctx) {
-                        new_fd = fdctx->fd;
-                        if (new_fd) {
-                                fd_unref (new_fd);
+                        activefd = fdctx->activefd;
+                        if (activefd) {
+                                fd_unref (activefd);
                         }
 
                         GF_FREE (fdctx);
@@ -3705,106 +3759,194 @@ out:
 
 
 int
-fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
-                 xlator_t *new_subvol)
+fuse_migrate_fd_open (xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                      xlator_t *old_subvol, xlator_t *new_subvol)
 {
-        int            ret                = -1;
-        loc_t          loc                = {0, };
-        char           create_in_progress = 0;
-        inode_t       *old_inode          = NULL;
-        int            flags              = 0;
+        loc_t          loc       = {0, };
+        fd_t          *newfd     = NULL, *old_activefd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        fuse_fd_ctx_t *newfd_ctx = NULL;
+        int            ret       = 0, flags = 0;
 
-        /* could've used pthread_cond_wait, but that requires a cond variable to
-         * be mainted for each fd and that is a bit too much overhead.
-         */
-        do {
-                LOCK (&fd->inode->lock);
-                {
-                        if (uuid_is_null (fd->inode->gfid)) {
-                                create_in_progress = 1;
-                        } else {
-                                create_in_progress = 0;
-                        }
-                }
-                UNLOCK (&fd->inode->lock);
+        ret = inode_path (basefd->inode, NULL, (char **)&loc.path);
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot construct path of gfid (%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                        uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
 
-                if (create_in_progress) {
-                        gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                                "create call on fd (%p) is in progress, "
-                                "hence waiting", fd);
-                        sleep (1);
-                }
+        uuid_copy (loc.gfid, basefd->inode->gfid);
 
-        } while (create_in_progress);
+        loc.inode = inode_find (new_subvol->itable, basefd->inode->gfid);
 
-        if (fd->inode->table->xl == old_subvol) {
-                ret = syncop_fsync (old_subvol, fd, 0);
+        if (loc.inode == NULL) {
+                ret = fuse_nameless_lookup (new_subvol, basefd->inode->gfid,
+                                            &loc);
                 if (ret < 0) {
                         gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "syncop_fsync failed (%s)", strerror (errno));
+                                "name-less lookup of gfid (%s) failed (%s)"
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                uuid_utoa (basefd->inode->gfid),
+                                strerror (errno),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                        goto out;
                 }
+
+        }
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        newfd = fd_create (loc.inode, basefd->pid);
+        if (newfd == NULL) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "cannot create new fd, hence not migrating basefd "
+                        "(ptr:%p inode-gfid:%s) "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (loc.inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
+
+        newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+
+        newfd_ctx = fuse_fd_ctx_check_n_create (this, newfd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", newfd_ctx, out);
+
+        if (IA_ISDIR (basefd->inode->ia_type)) {
+                ret = syncop_opendir (new_subvol, &loc, newfd);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fd (%p) was not "
-                        "migrated during previous graph switch", fd);
+                flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+                ret = syncop_open (new_subvol, &loc, flags, newfd);
         }
 
-        loc.path = "";
-        loc.name = NULL;
+        if (ret < 0) {
+                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                        "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid), strerror (errno),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+                goto out;
+        }
 
-        loc.inode = inode_find (new_subvol->itable, fd->inode->gfid);
+        fd_bind (newfd);
 
-        if (loc.inode == NULL) {
-                ret = fuse_nameless_lookup (new_subvol, fd->inode->gfid, &loc);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "name-less lookup of gfid (%s) failed (%s)",
-                                uuid_utoa (fd->inode->gfid), strerror (errno));
-                        goto out;
+        LOCK (&basefd->lock);
+        {
+                if (basefd_ctx->activefd != NULL) {
+                        old_activefd = basefd_ctx->activefd;
                 }
 
+                basefd_ctx->activefd = newfd;
         }
+        UNLOCK (&basefd->lock);
 
-        old_inode = fd->inode;
+        if (old_activefd != NULL) {
+                fd_unref (old_activefd);
+        }
 
-        inode_ref (loc.inode);
+        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                "migrated basefd (%p) to newfd (%p) "
+                "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd, newfd,
+                old_subvol->name, old_subvol->graph->id,
+                new_subvol->name, new_subvol->graph->id);
+
+        newfd = NULL;
+        ret = 0;
+out:
+        loc_wipe (&loc);
+
+        return ret;
+}
 
-        LOCK (&fd->inode->lock);
+
+int
+fuse_migrate_fd (xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
+                 xlator_t *new_subvol)
+{
+        int            ret                = -1;
+        char           create_in_progress = 0;
+        fuse_fd_ctx_t *basefd_ctx          = NULL;
+        fd_t          *oldfd              = NULL;
+
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
+
+        LOCK (&basefd->lock);
         {
-                list_del_init (&fd->inode_list);
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd
+                        : basefd;
+                fd_ref (oldfd);
         }
-        UNLOCK (&fd->inode->lock);
+        UNLOCK (&basefd->lock);
 
-        LOCK (&fd->lock);
+        LOCK (&oldfd->inode->lock);
         {
-                fd->inode = loc.inode;
+                if (uuid_is_null (oldfd->inode->gfid)) {
+                        create_in_progress = 1;
+                } else {
+                        create_in_progress = 0;
+                }
         }
-        UNLOCK (&fd->lock);
+        UNLOCK (&oldfd->inode->lock);
 
-        if (IA_ISDIR (fd->inode->ia_type)) {
-                ret = syncop_opendir (new_subvol, &loc, fd);
-        } else {
-                flags = fd->flags & ~(O_CREAT | O_EXCL);
-                ret = syncop_open (new_subvol, &loc, flags, fd);
+        if (create_in_progress) {
+                gf_log ("glusterfs-fuse", GF_LOG_INFO,
+                        "create call on fd (%p) is in progress "
+                        "(basefd-ptr:%p basefd-inode.gfid:%s), "
+                        "hence deferring migration till application does an "
+                        "fd based operation on this fd"
+                        "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
+                        oldfd, basefd, uuid_utoa (basefd->inode->gfid),
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
+
+                ret = 0;
+                goto out;
         }
 
-        if (ret < 0) {
+        if (oldfd->inode->table->xl == old_subvol) {
+                ret = syncop_fsync (old_subvol, oldfd, 0);
+                if (ret < 0) {
+                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
+                                "syncop_fsync failed (%s) on fd (%p)"
+                                "(basefd:%p basefd-inode.gfid:%s) "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                strerror (errno), oldfd, basefd,
+                                uuid_utoa (basefd->inode->gfid),
+                                old_subvol->name, old_subvol->graph->id,
+                                new_subvol->name, new_subvol->graph->id);
+                }
+        } else {
                 gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "open on gfid (%s) failed (%s)",
-                        uuid_utoa (fd->inode->gfid), strerror (errno));
-                goto out;
+                        "basefd (ptr:%p inode-gfid:%s) was not "
+                        "migrated during previous graph switch"
+                        "(old-subvolume:%s-%d new-subvolume: %s-%d)", basefd,
+                        basefd->inode->gfid,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
         }
 
-        fd_bind (fd);
-
-        ret = 0;
+        ret = fuse_migrate_fd_open (this, basefd, oldfd, old_subvol,
+                                    new_subvol);
 out:
-        if (loc.inode != NULL) {
-                inode_unref (loc.inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "migration of basefd "
+                        "(ptr:%p inode-gfid:%s) failed"
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        oldfd ? uuid_utoa (oldfd->inode->gfid) : NULL,
+                        old_subvol->name, old_subvol->graph->id,
+                        new_subvol->name, new_subvol->graph->id);
         }
 
-        if (old_inode != NULL) {
-                inode_unref (old_inode);
-        }
+        fd_unref (oldfd);
 
         return ret;
 }
@@ -3837,13 +3979,17 @@ fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
                         ret = fuse_migrate_fd (this, fd, old_subvol,
                                                new_subvol);
 
-                        fdctx = fuse_fd_ctx_check_n_create (this, fd);
+                        fdctx = fuse_fd_ctx_get (this, fd);
                         if (fdctx) {
-                                if (ret < 0) {
-                                        fdctx->migration_failed = 1;
-                                } else {
-                                        fdctx->migration_failed = 0;
+                                LOCK (&fd->lock);
+                                {
+                                        if (ret < 0) {
+                                                fdctx->migration_failed = 1;
+                                        } else {
+                                                fdctx->migration_failed = 0;
+                                        }
                                 }
+                                UNLOCK (&fd->lock);
                         }
                 }
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index c700554f9..fdfe08b2c 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -400,10 +400,10 @@ typedef struct {
         int32_t        fd_no;
 } fuse_state_t;
 
-typedef struct fuse_fd_ctx {
+typedef struct {
         uint32_t  open_flags;
         char      migration_failed;
-        fd_t     *fd;
+        fd_t     *activefd;
 } fuse_fd_ctx_t;
 
 typedef void (*fuse_resume_fn_t) (fuse_state_t *state);
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 6265a41d8..e7e580fe8 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -350,31 +350,49 @@ fuse_resolve_inode (fuse_state_t *state)
 int
 fuse_migrate_fd_task (void *data)
 {
-        int                     ret   = -1;
-        fuse_fd_ctx_t          *fdctx = NULL;
-        fuse_state_t           *state = NULL;
+        int            ret        = -1;
+        fuse_state_t  *state      = NULL;
+        fd_t          *basefd     = NULL, *oldfd = NULL;
+        fuse_fd_ctx_t *basefd_ctx = NULL;
+        xlator_t      *old_subvol = NULL;
 
         state = data;
         if (state == NULL) {
                 goto out;
         }
 
-        ret = fuse_migrate_fd (state->this, state->fd,
-                               state->fd->inode->table->xl,
+        basefd = state->fd;
+
+        basefd_ctx = fuse_fd_ctx_get (state->this, basefd);
+
+        LOCK (&basefd->lock);
+        {
+                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+                fd_ref (oldfd);
+        }
+        UNLOCK (&basefd->lock);
+
+        old_subvol = oldfd->inode->table->xl;
+
+        ret = fuse_migrate_fd (state->this, basefd, old_subvol,
                                state->active_subvol);
 
-        fdctx = fuse_fd_ctx_check_n_create (state->this, state->fd);
-        if (fdctx != NULL) {
+        LOCK (&basefd->lock);
+        {
                 if (ret < 0) {
-                        fdctx->migration_failed = 1;
+                        basefd_ctx->migration_failed = 1;
                 } else {
-                        fdctx->migration_failed = 0;
+                        basefd_ctx->migration_failed = 0;
                 }
         }
+        UNLOCK (&basefd->lock);
 
         ret = 0;
 
 out:
+        if (oldfd)
+                fd_unref (oldfd);
+
         return ret;
 }
 
@@ -395,22 +413,55 @@ fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
         return error;
 }
 
+#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                 \
+        do {                                                    \
+                LOCK (&basefd->lock);                           \
+                {                                               \
+                        activefd = basefd_ctx->activefd ?       \
+                                basefd_ctx->activefd : basefd;  \
+                        if (activefd != basefd) {               \
+                                fd_ref (activefd);              \
+                        }                                       \
+                }                                               \
+                UNLOCK (&basefd->lock);                         \
+                                                                \
+                if (activefd == basefd) {                       \
+                        fd_ref (activefd);                      \
+                }                                               \
+        } while (0);
+
 
 static int
 fuse_resolve_fd (fuse_state_t *state)
 {
-        fuse_resolve_t         *resolve            = NULL;
-	fd_t                   *fd                 = NULL;
-	xlator_t               *active_subvol      = NULL;
-        int                     ret                = 0;
-        char                    fd_migration_error = 0;
+        fuse_resolve_t *resolve            = NULL;
+	fd_t           *basefd             = NULL, *activefd = NULL;
+	xlator_t       *active_subvol      = NULL, *this = NULL;
+        int             ret                = 0;
+        char            fd_migration_error = 0;
+        fuse_fd_ctx_t  *basefd_ctx         = NULL;
 
         resolve = state->resolve_now;
 
-        fd = resolve->fd;
-	active_subvol = fd->inode->table->xl;
+        this = state->this;
+
+        basefd = resolve->fd;
+        basefd_ctx = fuse_fd_ctx_get (this, basefd);
+        if (basefd_ctx == NULL) {
+                gf_log (state->this->name, GF_LOG_WARNING,
+                        "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
+                        "resolver erroring out with errno EINVAL",
+                        basefd, uuid_utoa (basefd->inode->gfid));
+                resolve->op_ret = -1;
+                resolve->op_errno = EINVAL;
+                goto resolve_continue;
+        }
+
+        FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
 
-        fd_migration_error = fuse_migrate_fd_error (state->this, fd);
+        active_subvol = activefd->inode->table->xl;
+
+        fd_migration_error = fuse_migrate_fd_error (state->this, basefd);
         if (fd_migration_error) {
                 resolve->op_ret = -1;
                 resolve->op_errno = EBADF;
@@ -418,35 +469,78 @@ fuse_resolve_fd (fuse_state_t *state)
                 ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
                                     NULL, NULL, state);
 
-                fd_migration_error = fuse_migrate_fd_error (state->this, fd);
+                fd_migration_error = fuse_migrate_fd_error (state->this,
+                                                            basefd);
+                fd_unref (activefd);
+
+                FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
+                active_subvol = activefd->inode->table->xl;
 
                 if ((ret == -1) || fd_migration_error
-                    || (state->active_subvol != fd->inode->table->xl)) {
+                    || (state->active_subvol != active_subvol)) {
                         if (ret == -1) {
                                 gf_log (state->this->name, GF_LOG_WARNING,
-                                        "starting sync-task to migrate fd (%p)"
-                                        " failed", fd);
+                                        "starting sync-task to migrate "
+                                        "basefd (ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
                         } else {
                                 gf_log (state->this->name, GF_LOG_WARNING,
-                                        "fd migration of fd (%p) failed", fd);
+                                        "fd migration of basefd "
+                                        "(ptr:%p inode-gfid:%s) failed "
+                                        "(old-subvolume:%s-%d "
+                                        "new-subvolume:%s-%d)",
+                                        basefd,
+                                        uuid_utoa (basefd->inode->gfid),
+                                        active_subvol->name,
+                                        active_subvol->graph->id,
+                                        state->active_subvol->name,
+                                        state->active_subvol->graph->id);
                         }
 
                         resolve->op_ret = -1;
                         resolve->op_errno = EBADF;
                 } else {
                         gf_log (state->this->name, GF_LOG_DEBUG,
-                                "fd (%p) migrated successfully in resolver",
-                                fd);
+                                "basefd (ptr:%p inode-gfid:%s) migrated "
+                                "successfully in resolver "
+                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                                basefd, uuid_utoa (basefd->inode->gfid),
+                                active_subvol->name, active_subvol->graph->id,
+                                state->active_subvol->name,
+                                state->active_subvol->graph->id);
                 }
         }
 
         if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
-                gf_log ("fuse-resolve", GF_LOG_WARNING, "migration of fd (%p) "
-                        "did not complete, failing fop with EBADF", fd);
+                gf_log ("fuse-resolve", GF_LOG_WARNING,
+                        "migration of basefd (ptr:%p inode-gfid:%s) "
+                        "did not complete, failing fop with EBADF "
+                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
+                        uuid_utoa (basefd->inode->gfid),
+                        active_subvol->name, active_subvol->graph->id,
+                        state->active_subvol->name,
+                        state->active_subvol->graph->id);
+        }
+
+        if (activefd != basefd) {
+                state->fd = fd_ref (activefd);
+                fd_unref (basefd);
         }
 
 	/* state->active_subvol = active_subvol; */
 
+resolve_continue:
+        if (activefd != NULL) {
+                fd_unref (activefd);
+        }
+
         fuse_resolve_continue (state);
 
         return 0;