cluster/dht: Check for open fd only on EBADF

DHT fd based fops will now check if the fd is
open on the cached subvol only if the call fails
with EBADF.

This will improve performance for scenarios where
a rebalance is not running which would be most of
the time.

Change-Id: Idfaeb8927af769c6110d07a165a0fe2307369239
BUG: 1476665
Signed-off-by: N Balachandran <nbalacha@redhat.com>
Reviewed-on: https://review.gluster.org/17922
Smoke: Gluster Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>

2 files changed, 116 insertions, 160 deletions

diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 58a04302888..664c4adf4eb 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -173,6 +173,13 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1)
                 goto out;
 
@@ -377,8 +384,6 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
         dht_layout_t *layout = NULL;
         int           i = 0;
         int           call_cnt = 0;
-        int           ret      = -1;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -404,19 +409,10 @@ dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
                 local->call_cnt = 1;
 
                 subvol = local->cached_subvol;
-                if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                        STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,
-                                           subvol, subvol->fops->fstat, fd,
-                                           xdata);
-
-                } else {
-                        ret = dht_check_and_open_fd_on_subvol (this, frame);
-
-                        if (ret)
-                                goto err;
-                }
 
+                STACK_WIND_COOKIE (frame, dht_file_attr_cbk, subvol,
+                                   subvol, subvol->fops->fstat, fd,
+                                   xdata);
                 return 0;
         }
 
@@ -462,6 +458,13 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if ((op_ret == -1) && !dht_inode_missing(op_errno))
                 goto out;
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
 
@@ -545,7 +548,6 @@ dht_readv (call_frame_t *frame, xlator_t *this,
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -573,19 +575,10 @@ dht_readv (call_frame_t *frame, xlator_t *this,
         local->rebalance.flags  = flags;
         local->call_cnt = 1;
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                            local->fd, local->rebalance.size,
-                            local->rebalance.offset,
-                            local->rebalance.flags, local->xattr_req);
-
-        } else {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
-
+        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                    local->fd, local->rebalance.size,
+                    local->rebalance.offset,
+                    local->rebalance.flags, local->xattr_req);
 
         return 0;
 
@@ -743,6 +736,13 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         local->rebalance.target_op_fn = dht_flush2;
 
         local->op_ret = op_ret;
@@ -804,7 +804,6 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -829,18 +828,8 @@ dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 
         local->call_cnt = 1;
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND (frame, dht_flush_cbk,
-                            subvol, subvol->fops->flush, fd, local->xattr_req);
-                return 0;
-
-        } else {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
+        STACK_WIND (frame, dht_flush_cbk,
+                    subvol, subvol->fops->flush, fd, local->xattr_req);
         return 0;
 
 err:
@@ -874,6 +863,13 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1) {
                 if (local->stbuf.ia_blocks) {
                         dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
@@ -974,7 +970,6 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -994,20 +989,9 @@ dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
 
         subvol = local->cached_subvol;
 
-
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,
-                                   subvol->fops->fsync, local->fd,
-                                   local->rebalance.flags, local->xattr_req);
-
-        } else {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
-
-
+        STACK_WIND_COOKIE (frame, dht_fsync_cbk, subvol, subvol,
+                           subvol->fops->fsync, local->fd,
+                           local->rebalance.flags, local->xattr_req);
         return 0;
 
 err:
@@ -1301,8 +1285,7 @@ dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
         local->call_cnt = 1;
 
-        STACK_WIND (frame,
-                    dht_xattrop_cbk,
+        STACK_WIND (frame, dht_xattrop_cbk,
                     subvol, subvol->fops->xattrop,
                     loc, flags, dict, xdata);
 
@@ -1401,8 +1384,7 @@ dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
 
         local->call_cnt = 1;
 
-        STACK_WIND (frame,
-                    dht_inodelk_cbk,
+        STACK_WIND (frame, dht_inodelk_cbk,
                     lock_subvol, lock_subvol->fops->inodelk,
                     volume, loc, cmd, lock, xdata);
 
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index f925d4b59ed..954a914e9cd 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -44,6 +44,13 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (op_ret == -1 && !dht_inode_missing(op_errno)) {
                 local->op_errno = op_errno;
                 local->op_ret = -1;
@@ -162,7 +169,6 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         xlator_t     *subvol   = NULL;
         int           op_errno = -1;
         dht_local_t  *local    = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -183,7 +189,6 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto err;
         }
 
-
         if (xdata)
                 local->xattr_req = dict_ref (xdata);
 
@@ -194,22 +199,13 @@ dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         local->rebalance.iobref = iobref_ref (iobref);
         local->call_cnt = 1;
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,
-                                   subvol->fops->writev, fd,
-                                   local->rebalance.vector,
-                                   local->rebalance.count,
-                                   local->rebalance.offset,
-                                   local->rebalance.flags,
-                                   local->rebalance.iobref, local->xattr_req);
-                return 0;
-
-        } else {
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
+        STACK_WIND_COOKIE (frame, dht_writev_cbk, subvol, subvol,
+                           subvol->fops->writev, fd,
+                           local->rebalance.vector,
+                           local->rebalance.count,
+                           local->rebalance.offset,
+                           local->rebalance.flags,
+                           local->rebalance.iobref, local->xattr_req);
 
         return 0;
 
@@ -253,6 +249,14 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        /* Can only occur for ftruncate */
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1) {
                 if (local->stbuf.ia_blocks) {
                         dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
@@ -408,7 +412,6 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret   = -1;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -434,20 +437,9 @@ dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         if (xdata)
                 local->xattr_req = dict_ref (xdata);
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
-                                   subvol->fops->ftruncate, fd,
-                                   local->rebalance.offset, local->xattr_req);
-                return 0;
-
-        } else {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
-
+        STACK_WIND_COOKIE (frame, dht_truncate_cbk, subvol, subvol,
+                           subvol->fops->ftruncate, fd,
+                           local->rebalance.offset, local->xattr_req);
         return 0;
 
 err:
@@ -487,6 +479,13 @@ dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1) {
                 if (local->stbuf.ia_blocks) {
                         dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
@@ -586,7 +585,6 @@ dht_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -614,22 +612,12 @@ dht_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
         if (xdata)
                 local->xattr_req = dict_ref (xdata);
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,
-                                   subvol->fops->fallocate, fd,
-                                   local->rebalance.flags,
-                                   local->rebalance.offset,
-                                   local->rebalance.size,
-                                   local->xattr_req);
-                return 0;
-
-        } else {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
+        STACK_WIND_COOKIE (frame, dht_fallocate_cbk, subvol, subvol,
+                           subvol->fops->fallocate, fd,
+                           local->rebalance.flags,
+                           local->rebalance.offset,
+                           local->rebalance.size,
+                           local->xattr_req);
 
         return 0;
 
@@ -670,6 +658,13 @@ dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1) {
                 if (local->stbuf.ia_blocks) {
                         dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
@@ -769,7 +764,6 @@ dht_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
-        int           ret      = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -796,22 +790,11 @@ dht_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         if (xdata)
                 local->xattr_req = dict_ref (xdata);
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,
-                                   subvol->fops->discard, fd,
-                                   local->rebalance.offset,
-                                   local->rebalance.size,
-                                   local->xattr_req);
-                return 0;
-
-        } else {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-
-        }
+        STACK_WIND_COOKIE (frame, dht_discard_cbk, subvol, subvol,
+                           subvol->fops->discard, fd,
+                           local->rebalance.offset,
+                           local->rebalance.size,
+                           local->xattr_req);
 
         return 0;
 
@@ -849,6 +832,13 @@ dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         if (local->call_cnt != 1) {
                 if (local->stbuf.ia_blocks) {
                         dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
@@ -952,7 +942,6 @@ dht_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         xlator_t     *subvol       = NULL;
         int           op_errno     = -1;
         dht_local_t  *local        = NULL;
-        int           ret          = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -979,21 +968,10 @@ dht_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
         if (xdata)
                 local->xattr_req = dict_ref (xdata);
 
-        if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,
-                                   subvol->fops->zerofill, fd,
-                                   local->rebalance.offset,
-                                   local->rebalance.size, local->xattr_req);
-                return 0;
-
-        } else {
-
-                ret = dht_check_and_open_fd_on_subvol (this, frame);
-                if (ret)
-                        goto err;
-        }
-
+        STACK_WIND_COOKIE (frame, dht_zerofill_cbk, subvol, subvol,
+                           subvol->fops->zerofill, fd,
+                           local->rebalance.offset,
+                           local->rebalance.size, local->xattr_req);
 
         return 0;
 
@@ -1027,9 +1005,17 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
+
         if (local->call_cnt != 1)
                 goto out;
 
+        if (op_ret == -1 && (op_errno == EBADF)) {
+                ret = dht_check_and_open_fd_on_subvol (this, frame);
+                if (ret)
+                        goto out;
+                return 0;
+        }
+
         local->op_ret = op_ret;
         local->op_errno = op_errno;
 
@@ -1241,8 +1227,6 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
         int           op_errno = -1;
         int           i = -1;
         int           call_cnt = 0;
-        int           ret      = -1;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -1279,21 +1263,11 @@ dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
                 local->call_cnt = 1;
                 subvol = local->cached_subvol;
 
-                if (dht_fd_open_on_dst (this, fd, subvol)) {
-
-                        STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
-                                           subvol, subvol->fops->fsetattr, fd,
-                                           &local->rebalance.stbuf,
-                                           local->rebalance.flags,
-                                           local->xattr_req);
-                        return 0;
-
-                } else {
-                        ret = dht_check_and_open_fd_on_subvol (this, frame);
-                        if (ret)
-                                goto err;
-
-                }
+                STACK_WIND_COOKIE (frame, dht_file_setattr_cbk, subvol,
+                                   subvol, subvol->fops->fsetattr, fd,
+                                   &local->rebalance.stbuf,
+                                   local->rebalance.flags,
+                                   local->xattr_req);
                 return 0;
         }