Posix: Optimize posix code to improve file creation

Problem: Before executing a fop in POSIX xlator it builds an internal
         path based on GFID.To validate the path it call's (l)stat
         system call and while .glusterfs is heavily loaded kernel takes
         time to lookup inode and due to that performance drops

Solution: In this patch we followed two ways to improve the performance.
          1) Keep open fd specific to first level directory(gfid[0])
             in .glusterfs, it would force to kernel keep the inodes
             from all those files in cache. In case of memory pressure
             kernel won't uncache first level inodes. We need to open
             256 fd's per brick to access the entry faster.
          2) Use at based call's to access relative path to reduce
             path based lookup time.

Note: To verify the patch we have executed kernel untar 100 times on 6
      different clients after enabling metadata group-cache and some
      other option.We were getting more than 20 percent improvement in
      kenel untar after applying the patch.

Credits: Xavi Hernandez <xhernandez@redhat.com>
Change-Id: I1643e6b01ed669b2bb148d02f4e6a8e08da45343
updates: #891
Signed-off-by: Mohit Agrawal <moagrawal@redhat.com>

9 files changed, 255 insertions, 75 deletions

diff --git a/libglusterfs/src/glusterfs/syscall.h b/libglusterfs/src/glusterfs/syscall.h
index 6b33c141a5e..91e921aea50 100644
--- a/libglusterfs/src/glusterfs/syscall.h
+++ b/libglusterfs/src/glusterfs/syscall.h
@@ -96,18 +96,27 @@ int
 sys_unlink(const char *pathname);
 
 int
+sys_unlinkat(int dfd, const char *pathname);
+
+int
 sys_rmdir(const char *pathname);
 
 int
 sys_symlink(const char *oldpath, const char *newpath);
 
 int
+sys_symlinkat(const char *oldpath, int dirfd, const char *newpath);
+
+int
 sys_rename(const char *oldpath, const char *newpath);
 
 int
 sys_link(const char *oldpath, const char *newpath);
 
 int
+sys_linkat(int oldfd, const char *oldpath, int newfd, const char *newpath);
+
+int
 sys_chmod(const char *path, mode_t mode);
 
 int
diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym
index dc7382ba749..47e0872ca59 100644
--- a/libglusterfs/src/libglusterfs.sym
+++ b/libglusterfs/src/libglusterfs.sym
@@ -1039,6 +1039,7 @@ sys_futimes
 sys_lchown
 sys_lgetxattr
 sys_link
+sys_linkat
 sys_llistxattr
 sys_lremovexattr
 sys_lseek
@@ -1062,8 +1063,10 @@ sys_rmdir
 sys_stat
 sys_statvfs
 sys_symlink
+sys_symlinkat
 sys_truncate
 sys_unlink
+sys_unlinkat
 sys_utimensat
 sys_write
 sys_writev
diff --git a/libglusterfs/src/syscall.c b/libglusterfs/src/syscall.c
index 1d88c8adac1..49f782a837a 100644
--- a/libglusterfs/src/syscall.c
+++ b/libglusterfs/src/syscall.c
@@ -214,6 +214,15 @@ sys_unlink(const char *pathname)
 }
 
 int
+sys_unlinkat(int dfd, const char *pathname)
+{
+#ifdef GF_SOLARIS_HOST_OS
+    return FS_RET_CHECK0(solaris_unlinkat(dfd, pathname, 0), errno);
+#endif
+    return FS_RET_CHECK0(unlinkat(dfd, pathname, 0), errno);
+}
+
+int
 sys_rmdir(const char *pathname)
 {
     return FS_RET_CHECK0(rmdir(pathname), errno);
@@ -226,6 +235,12 @@ sys_symlink(const char *oldpath, const char *newpath)
 }
 
 int
+sys_symlinkat(const char *oldpath, int dirfd, const char *newpath)
+{
+    return FS_RET_CHECK0(symlinkat(oldpath, dirfd, newpath), errno);
+}
+
+int
 sys_rename(const char *oldpath, const char *newpath)
 {
 #ifdef GF_SOLARIS_HOST_OS
@@ -253,6 +268,12 @@ sys_link(const char *oldpath, const char *newpath)
 }
 
 int
+sys_linkat(int oldfd, const char *oldpath, int newfd, const char *newpath)
+{
+    return FS_RET_CHECK0(linkat(oldfd, oldpath, newfd, newpath, 0), errno);
+}
+
+int
 sys_chmod(const char *path, mode_t mode)
 {
     return FS_RET_CHECK0(chmod(path, mode), errno);
diff --git a/tests/basic/ec/gfapi-ec-open-truncate.c b/tests/basic/ec/gfapi-ec-open-truncate.c
index 22f17f436b6..fb16807003a 100644
--- a/tests/basic/ec/gfapi-ec-open-truncate.c
+++ b/tests/basic/ec/gfapi-ec-open-truncate.c
@@ -150,7 +150,8 @@ main(int argc, char *argv[])
     for (i = 0; i < 20; i++) {
         ret = system(
             "[ $(for i in $(pgrep glusterfsd); do ls -l /proc/$i/fd | grep "
-            "\"[.]glusterfs\" | grep -v health_check; done | wc -l) == 3 ]");
+            "\"[.]glusterfs\" | grep -v \".glusterfs/[0-9a-f][0-9a-f]\" | grep "
+            "-v health_check; done | wc -l) == 3 ]");
         if (WIFEXITED(ret) && WEXITSTATUS(ret)) {
             printf("Ret value of system: %d\n, ifexited: %d, exitstatus: %d",
                    ret, WIFEXITED(ret), WEXITSTATUS(ret));
diff --git a/xlators/storage/posix/src/posix-common.c b/xlators/storage/posix/src/posix-common.c
index 20a8d231181..1fc0eae2056 100644
--- a/xlators/storage/posix/src/posix-common.c
+++ b/xlators/storage/posix/src/posix-common.c
@@ -543,6 +543,30 @@ posix_create_unlink_dir(xlator_t *this)
     return 0;
 }
 
+int
+posix_create_open_directory_based_fd(xlator_t *this, int pdirfd, char *dir_name)
+{
+    int ret = -1;
+
+    ret = sys_openat(pdirfd, dir_name, (O_DIRECTORY | O_RDONLY), 0);
+    if (ret < 0 && errno == ENOENT) {
+        ret = sys_mkdirat(pdirfd, dir_name, 0700);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
+                   "Creating directory %s failed", dir_name);
+            goto out;
+        }
+        ret = sys_openat(pdirfd, dir_name, (O_DIRECTORY | O_RDONLY), 0);
+        if (ret < 0 && errno != EEXIST) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
+                   "error mkdir hash-1 %s ", dir_name);
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
 /**
  * init -
  */
@@ -579,6 +603,14 @@ posix_init(xlator_t *this)
     int force_directory = -1;
     int create_mask = -1;
     int create_directory_mask = -1;
+    char dir_handle[PATH_MAX] = {
+        0,
+    };
+    int i;
+    char fhash[4] = {
+        0,
+    };
+    int hdirfd = -1;
 
     dir_data = dict_get(this->options, "directory");
 
@@ -621,6 +653,11 @@ posix_init(xlator_t *this)
     _private->base_path = gf_strdup(dir_data->data);
     _private->base_path_length = dir_data->len - 1;
 
+    _private->dirfd = -1;
+    _private->mount_lock = -1;
+    for (i = 0; i < 256; i++)
+        _private->arrdfd[i] = -1;
+
     ret = dict_get_str(this->options, "hostname", &_private->hostname);
     if (ret) {
         _private->hostname = GF_CALLOC(256, sizeof(char), gf_common_mt_char);
@@ -893,8 +930,9 @@ posix_init(xlator_t *this)
     /* performing open dir on brick dir locks the brick dir
      * and prevents it from being unmounted
      */
-    _private->mount_lock = sys_opendir(dir_data->data);
-    if (!_private->mount_lock) {
+    _private->mount_lock = sys_open(dir_data->data, (O_DIRECTORY | O_RDONLY),
+                                    0);
+    if (_private->mount_lock < 0) {
         ret = -1;
         op_errno = errno;
         gf_msg(this->name, GF_LOG_ERROR, 0, P_MSG_DIR_OPERATION_FAILED,
@@ -938,6 +976,28 @@ posix_init(xlator_t *this)
     }
 
     this->private = (void *)_private;
+    snprintf(dir_handle, sizeof(dir_handle), "%s/%s", _private->base_path,
+             GF_HIDDEN_PATH);
+    hdirfd = posix_create_open_directory_based_fd(this, _private->mount_lock,
+                                                  dir_handle);
+    if (hdirfd < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
+               "error open directory failed for dir %s", dir_handle);
+        ret = -1;
+        goto out;
+    }
+    _private->dirfd = hdirfd;
+    for (i = 0; i < 256; i++) {
+        snprintf(fhash, sizeof(fhash), "%02x", i);
+        _private->arrdfd[i] = posix_create_open_directory_based_fd(this, hdirfd,
+                                                                   fhash);
+        if (_private->arrdfd[i] < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
+                   "error openat failed for file %s", fhash);
+            ret = -1;
+            goto out;
+        }
+    }
 
     op_ret = posix_handle_init(this);
     if (op_ret == -1) {
@@ -1101,9 +1161,27 @@ posix_init(xlator_t *this)
                    out);
 
     GF_OPTION_INIT("ctime", _private->ctime, bool, out);
+
 out:
     if (ret) {
         if (_private) {
+            if (_private->dirfd >= 0) {
+                sys_close(_private->dirfd);
+                _private->dirfd = -1;
+            }
+
+            for (i = 0; i < 256; i++) {
+                if (_private->arrdfd[i] >= 0) {
+                    sys_close(_private->arrdfd[i]);
+                    _private->arrdfd[i] = -1;
+                }
+            }
+            /*unlock brick dir*/
+            if (_private->mount_lock >= 0) {
+                (void)sys_close(_private->mount_lock);
+                _private->mount_lock = -1;
+            }
+
             GF_FREE(_private->base_path);
 
             GF_FREE(_private->hostname);
@@ -1124,6 +1202,7 @@ posix_fini(xlator_t *this)
     struct posix_private *priv = this->private;
     gf_boolean_t health_check = _gf_false;
     int ret = 0;
+    int i = 0;
 
     if (!priv)
         return;
@@ -1134,6 +1213,18 @@ posix_fini(xlator_t *this)
     }
     UNLOCK(&priv->lock);
 
+    if (priv->dirfd >= 0) {
+        sys_close(priv->dirfd);
+        priv->dirfd = -1;
+    }
+
+    for (i = 0; i < 256; i++) {
+        if (priv->arrdfd[i] >= 0) {
+            sys_close(priv->arrdfd[i]);
+            priv->arrdfd[i] = -1;
+        }
+    }
+
     if (health_check) {
         (void)gf_thread_cleanup_xint(priv->health_check);
         priv->health_check = 0;
@@ -1161,8 +1252,10 @@ posix_fini(xlator_t *this)
         priv->fsyncer = 0;
     }
     /*unlock brick dir*/
-    if (priv->mount_lock)
-        (void)sys_closedir(priv->mount_lock);
+    if (priv->mount_lock >= 0) {
+        (void)sys_close(priv->mount_lock);
+        priv->mount_lock = -1;
+    }
 
     GF_FREE(priv->base_path);
     LOCK_DESTROY(&priv->lock);
diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c
index 667fec7d62f..c3a20f4c85c 100644
--- a/xlators/storage/posix/src/posix-entry-ops.c
+++ b/xlators/storage/posix/src/posix-entry-ops.c
@@ -176,6 +176,7 @@ posix_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
     struct posix_private *priv = NULL;
     posix_inode_ctx_t *ctx = NULL;
     int ret = 0;
+    int dfd = -1;
 
     VALIDATE_OR_GOTO(frame, out);
     VALIDATE_OR_GOTO(this, out);
@@ -232,12 +233,12 @@ posix_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
             if (!op_errno)
                 op_errno = ESTALE;
             loc_gfid(loc, gfid);
-            MAKE_HANDLE_ABSPATH(gfid_path, this, gfid);
-            ret = sys_stat(gfid_path, &statbuf);
+            MAKE_HANDLE_ABSPATH_FD(gfid_path, this, gfid, dfd);
+            ret = sys_fstatat(dfd, gfid_path, &statbuf, 0);
             if (ret == 0 && ((statbuf.st_mode & S_IFMT) == S_IFDIR))
                 /*Don't unset if it was a symlink to a dir.*/
                 goto parent;
-            ret = sys_lstat(gfid_path, &statbuf);
+            ret = sys_fstatat(dfd, gfid_path, &statbuf, AT_SYMLINK_NOFOLLOW);
             if (ret == 0 && statbuf.st_nlink == 1) {
                 gf_msg(this->name, GF_LOG_WARNING, op_errno,
                        P_MSG_HANDLE_DELETE,
diff --git a/xlators/storage/posix/src/posix-handle.c b/xlators/storage/posix/src/posix-handle.c
index f58f5416ff0..410b38da8cb 100644
--- a/xlators/storage/posix/src/posix-handle.c
+++ b/xlators/storage/posix/src/posix-handle.c
@@ -25,7 +25,7 @@
 #include <glusterfs/compat-errno.h>
 
 int
-posix_handle_mkdir_hashes(xlator_t *this, const char *newpath);
+posix_handle_mkdir_hashes(xlator_t *this, int dfd, uuid_t gfid);
 
 inode_t *
 posix_resolve(xlator_t *this, inode_table_t *itable, inode_t *parent,
@@ -331,9 +331,23 @@ posix_handle_pump(xlator_t *this, char *buf, int len, int maxlen,
     int ret = 0;
     int blen = 0;
     int link_len = 0;
+    char tmpstr[POSIX_GFID_HASH2_LEN] = {
+        0,
+    };
+    char d2[3] = {
+        0,
+    };
+    int index = 0;
+    int dirfd = 0;
+    struct posix_private *priv = this->private;
+
+    strncpy(tmpstr, (base_str + pfx_len + 3), 40);
+    strncpy(d2, (base_str + pfx_len), 2);
+    index = strtoul(d2, NULL, 16);
+    dirfd = priv->arrdfd[index];
 
     /* is a directory's symlink-handle */
-    ret = sys_readlink(base_str, linkname, 512);
+    ret = readlinkat(dirfd, tmpstr, linkname, 512);
     if (ret == -1) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_READLINK_FAILED,
                "internal readlink failed on %s ", base_str);
@@ -398,6 +412,11 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf,
     int pfx_len;
     int maxlen;
     char *buf;
+    int index = 0;
+    int dfd = 0;
+    char newstr[POSIX_GFID_HASH2_LEN] = {
+        0,
+    };
 
     priv = this->private;
 
@@ -411,12 +430,14 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf,
         buf = alloca(maxlen);
     }
 
+    index = gfid[0];
+    dfd = priv->arrdfd[index];
+
     base_len = (priv->base_path_length + SLEN(GF_HIDDEN_PATH) + 45);
     base_str = alloca(base_len + 1);
     base_len = snprintf(base_str, base_len + 1, "%s/%s/%02x/%02x/%s",
                         priv->base_path, GF_HIDDEN_PATH, gfid[0], gfid[1],
                         uuid_str);
-
     pfx_len = priv->base_path_length + 1 + SLEN(GF_HIDDEN_PATH) + 1;
 
     if (basename) {
@@ -425,7 +446,8 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf,
         len = snprintf(buf, maxlen, "%s", base_str);
     }
 
-    ret = sys_lstat(base_str, &stat);
+    snprintf(newstr, sizeof(newstr), "%02x/%s", gfid[1], uuid_str);
+    ret = sys_fstatat(dfd, newstr, &stat, AT_SYMLINK_NOFOLLOW);
 
     if (!(ret == 0 && S_ISLNK(stat.st_mode) && stat.st_nlink == 1))
         goto out;
@@ -438,7 +460,6 @@ posix_handle_path(xlator_t *this, uuid_t gfid, const char *basename, char *ubuf,
 
         if (ret == -1)
             break;
-
         ret = sys_lstat(buf, &stat);
     } while ((ret == -1) && errno == ELOOP);
 
@@ -485,6 +506,7 @@ posix_handle_init(xlator_t *this)
     struct stat exportbuf;
     char *rootstr = NULL;
     static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    int dfd = 0;
 
     priv = this->private;
 
@@ -534,9 +556,8 @@ posix_handle_init(xlator_t *this)
         return -1;
     }
 
-    MAKE_HANDLE_ABSPATH(rootstr, this, gfid);
-
-    ret = sys_stat(rootstr, &rootbuf);
+    MAKE_HANDLE_ABSPATH_FD(rootstr, this, gfid, dfd);
+    ret = sys_fstatat(dfd, rootstr, &rootbuf, 0);
     switch (ret) {
         case -1:
             if (errno != ENOENT) {
@@ -544,15 +565,14 @@ posix_handle_init(xlator_t *this)
                        "%s", priv->base_path);
                 return -1;
             }
-
-            ret = posix_handle_mkdir_hashes(this, rootstr);
+            ret = posix_handle_mkdir_hashes(this, dfd, gfid);
             if (ret) {
                 gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
                        "mkdir %s failed", rootstr);
                 return -1;
             }
 
-            ret = sys_symlink("../../..", rootstr);
+            ret = sys_symlinkat("../../..", dfd, rootstr);
             if (ret) {
                 gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
                        "symlink %s creation failed", rootstr);
@@ -681,30 +701,18 @@ out:
 }
 
 int
-posix_handle_mkdir_hashes(xlator_t *this, const char *newpath)
+posix_handle_mkdir_hashes(xlator_t *this, int dirfd, uuid_t gfid)
 {
-    char *duppath = NULL;
-    char *parpath = NULL;
-    int ret = 0;
-
-    duppath = strdupa(newpath);
-    parpath = dirname(duppath);
-    parpath = dirname(duppath);
-
-    ret = sys_mkdir(parpath, 0700);
-    if (ret == -1 && errno != EEXIST) {
-        gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
-               "error mkdir hash-1 %s ", parpath);
-        return -1;
-    }
-
-    strcpy(duppath, newpath);
-    parpath = dirname(duppath);
+    int ret = -1;
+    char d2[3] = {
+        0,
+    };
 
-    ret = sys_mkdir(parpath, 0700);
+    snprintf(d2, sizeof(d2), "%02x", gfid[1]);
+    ret = sys_mkdirat(dirfd, d2, 0700);
     if (ret == -1 && errno != EEXIST) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_HANDLE_CREATE,
-               "error mkdir hash-2 %s ", parpath);
+               "error mkdir hash-2 %s ", uuid_utoa(gfid));
         return -1;
     }
 
@@ -715,51 +723,59 @@ int
 posix_handle_hard(xlator_t *this, const char *oldpath, uuid_t gfid,
                   struct stat *oldbuf)
 {
-    char *newpath = NULL;
     struct stat newbuf;
+    struct stat hashbuf;
     int ret = -1;
     gf_boolean_t link_exists = _gf_false;
+    char d2[3] = {
+        0,
+    };
+    int dfd = -1;
+    char *newstr = NULL;
 
-    MAKE_HANDLE_ABSPATH(newpath, this, gfid);
+    MAKE_HANDLE_ABSPATH_FD(newstr, this, gfid, dfd);
+    ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW);
 
-    ret = sys_lstat(newpath, &newbuf);
     if (ret == -1 && errno != ENOENT) {
         gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "%s",
-               newpath);
+               uuid_utoa(gfid));
         return -1;
     }
 
     if (ret == -1 && errno == ENOENT) {
-        ret = posix_handle_mkdir_hashes(this, newpath);
+        snprintf(d2, sizeof(d2), "%02x", gfid[1]);
+        ret = sys_fstatat(dfd, d2, &hashbuf, 0);
         if (ret) {
-            gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-                   "mkdir %s failed ", newpath);
-            return -1;
+            ret = posix_handle_mkdir_hashes(this, dfd, gfid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
+                       "mkdir %s failed ", uuid_utoa(gfid));
+                return -1;
+            }
         }
-
-        ret = sys_link(oldpath, newpath);
+        ret = sys_linkat(AT_FDCWD, oldpath, dfd, newstr);
 
         if (ret) {
             if (errno != EEXIST) {
                 gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
                        "link %s -> %s"
                        "failed ",
-                       oldpath, newpath);
+                       oldpath, newstr);
                 return -1;
             } else {
                 link_exists = _gf_true;
             }
         }
+        ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW);
 
-        ret = sys_lstat(newpath, &newbuf);
         if (ret) {
             gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-                   "lstat on %s failed", newpath);
+                   "lstat on %s failed", uuid_utoa(gfid));
             return -1;
         }
         if ((link_exists) && (!S_ISREG(newbuf.st_mode))) {
             gf_msg(this->name, GF_LOG_ERROR, EINVAL, P_MSG_HANDLE_CREATE,
-                   "%s - Expected regular file", newpath);
+                   "%s - Expected regular file", uuid_utoa(gfid));
             return -1;
         }
     }
@@ -769,7 +785,8 @@ posix_handle_hard(xlator_t *this, const char *oldpath, uuid_t gfid,
                "mismatching ino/dev between file %s (%lld/%lld) "
                "and handle %s (%lld/%lld)",
                oldpath, (long long)oldbuf->st_ino, (long long)oldbuf->st_dev,
-               newpath, (long long)newbuf.st_ino, (long long)newbuf.st_dev);
+               uuid_utoa(gfid), (long long)newbuf.st_ino,
+               (long long)newbuf.st_dev);
         ret = -1;
     }
 
@@ -783,15 +800,23 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc,
     char *oldpath = NULL;
     char *newpath = NULL;
     struct stat newbuf;
+    struct stat hashbuf;
     int ret = -1;
+    char d2[3] = {
+        0,
+    };
+    int dfd = -1;
+    char *newstr = NULL;
 
     MAKE_HANDLE_ABSPATH(newpath, this, gfid);
+    MAKE_HANDLE_ABSPATH_FD(newstr, this, gfid, dfd);
     MAKE_HANDLE_RELPATH(oldpath, this, loc->pargfid, loc->name);
 
-    ret = sys_lstat(newpath, &newbuf);
+    ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW);
+
     if (ret == -1 && errno != ENOENT) {
         gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE, "%s",
-               newpath);
+               newstr);
         return -1;
     }
 
@@ -801,24 +826,30 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc,
             errno = EINVAL;
             return -1;
         }
-        ret = posix_handle_mkdir_hashes(this, newpath);
+
+        snprintf(d2, sizeof(d2), "%02x", gfid[1]);
+        ret = sys_fstatat(dfd, d2, &hashbuf, 0);
+
         if (ret) {
-            gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-                   "mkdir %s failed ", newpath);
-            return -1;
+            ret = posix_handle_mkdir_hashes(this, dfd, gfid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
+                       "mkdir %s failed ", newstr);
+                return -1;
+            }
         }
-
-        ret = sys_symlink(oldpath, newpath);
+        ret = sys_symlinkat(oldpath, dfd, newstr);
         if (ret) {
             gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-                   "symlink %s -> %s failed", oldpath, newpath);
+                   "symlink %s -> %s failed", oldpath, newstr);
             return -1;
         }
 
-        ret = sys_lstat(newpath, &newbuf);
+        ret = sys_fstatat(dfd, newstr, &newbuf, AT_SYMLINK_NOFOLLOW);
+
         if (ret) {
             gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-                   "stat on %s failed ", newpath);
+                   "stat on %s failed ", newstr);
             return -1;
         }
     }
@@ -826,7 +857,7 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc,
     ret = sys_stat(real_path, &newbuf);
     if (ret) {
         gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_CREATE,
-               "stat on %s failed ", newpath);
+               "stat on %s failed ", real_path);
         return -1;
     }
 
@@ -848,26 +879,33 @@ posix_handle_soft(xlator_t *this, const char *real_path, loc_t *loc,
 int
 posix_handle_unset_gfid(xlator_t *this, uuid_t gfid)
 {
-    char *path = NULL;
-    int ret = -1;
+    int ret = 0;
     struct stat stat;
+    int index = 0;
+    int dfd = 0;
+    char newstr[POSIX_GFID_HASH2_LEN] = {
+        0,
+    };
+    struct posix_private *priv = this->private;
 
-    MAKE_HANDLE_GFID_PATH(path, this, gfid);
+    index = gfid[0];
+    dfd = priv->arrdfd[index];
 
-    ret = sys_lstat(path, &stat);
+    snprintf(newstr, sizeof(newstr), "%02x/%s", gfid[1], uuid_utoa(gfid));
+    ret = sys_fstatat(dfd, newstr, &stat, AT_SYMLINK_NOFOLLOW);
 
     if (ret == -1) {
         if (errno != ENOENT) {
             gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_DELETE, "%s",
-                   path);
+                   newstr);
         }
         goto out;
     }
 
-    ret = sys_unlink(path);
-    if (ret == -1) {
+    ret = sys_unlinkat(dfd, newstr);
+    if (ret) {
         gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_HANDLE_DELETE,
-               "unlink %s failed ", path);
+               "unlink %s is failed", newstr);
     }
 
 out:
diff --git a/xlators/storage/posix/src/posix-handle.h b/xlators/storage/posix/src/posix-handle.h
index 70c68c3d89c..f33ed92620d 100644
--- a/xlators/storage/posix/src/posix-handle.h
+++ b/xlators/storage/posix/src/posix-handle.h
@@ -141,6 +141,16 @@
                  __priv->base_path, gfid[0], gfid[1], uuid_utoa(gfid));        \
     } while (0)
 
+#define MAKE_HANDLE_ABSPATH_FD(var, this, gfid, dfd)                           \
+    do {                                                                       \
+        struct posix_private *__priv = this->private;                          \
+        int findex = gfid[0];                                                  \
+        int __len = POSIX_GFID_HASH2_LEN;                                      \
+        var = alloca(__len);                                                   \
+        snprintf(var, __len, "%02x/%s", gfid[1], uuid_utoa(gfid));             \
+        dfd = __priv->arrdfd[findex];                                          \
+    } while (0)
+
 #define MAKE_ENTRY_HANDLE(entp, parp, this, loc, ent_p)                        \
     do {                                                                       \
         char *__parp;                                                          \
@@ -184,6 +194,7 @@
         /* expand ELOOP */                                                     \
     } while (0)
 
+#define POSIX_GFID_HASH2_LEN 45
 int
 posix_handle_gfid_path(xlator_t *this, uuid_t gfid, char *buf, size_t len);
 
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index fb609dff731..662b5c69f8a 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -152,7 +152,7 @@ struct posix_private {
 
     char *trash_path;
     /* lock for brick dir */
-    DIR *mount_lock;
+    int mount_lock;
 
     struct stat handledir;
 
@@ -214,6 +214,8 @@ struct posix_private {
     mode_t create_mask;
     mode_t create_directory_mask;
     uint32_t max_hardlinks;
+    int32_t arrdfd[256];
+    int dirfd;
 
     /* This option is used for either to call a landfill_purge or not */
     gf_boolean_t disable_landfill_purge;
@@ -331,6 +333,7 @@ posix_istat(xlator_t *this, inode_t *inode, uuid_t gfid, const char *basename,
 int
 posix_pstat(xlator_t *this, inode_t *inode, uuid_t gfid, const char *real_path,
             struct iatt *iatt, gf_boolean_t inode_locked);
+
 dict_t *
 posix_xattr_fill(xlator_t *this, const char *path, loc_t *loc, fd_t *fd,
                  int fdnum, dict_t *xattr, struct iatt *buf);