storage/posix: Implement .unlink directory

Problem: For EC volume, If a file descriptor is open and
file has been unlinked, any further write on that fd will
fail. When a write request comes, EC internally reads some
blocks using anonymous fd. This read will fail as the file
has already been unlinked.

Solution: To solve this issue, we are using .unlink directory
to keep track of unlinked file. If a file is to be unlinked
while its fd is open, move this to .unlink directory and unlink
it from .glusterfs and real path. Once all the fd will be closed,
remove this entry form .unlink directory.

master -
http://review.gluster.org/#/c/12816/

Change-Id: I8344edb0d340bdb883dc46458c16edbc336916b9
BUG: 1291557
Signed-off-by: Ashish Pandey <aspandey@redhat.com>
Reviewed-on: http://review.gluster.org/12968
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Tested-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Smoke: Gluster Build System <jenkins@build.gluster.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>

5 files changed, 276 insertions, 14 deletions

diff --git a/xlators/storage/posix/src/posix-handle.c b/xlators/storage/posix/src/posix-handle.c
index 96e7098d153..4aee2c1104c 100644
--- a/xlators/storage/posix/src/posix-handle.c
+++ b/xlators/storage/posix/src/posix-handle.c
@@ -21,6 +21,8 @@
 #include <alloca.h>
 #endif
 
+#include "common-utils.h"
+
 #include "posix-handle.h"
 #include "posix.h"
 #include "xlator.h"
@@ -448,7 +450,6 @@ out:
         return len;
 }
 
-
 int
 posix_handle_init (xlator_t *this)
 {
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index b6f3bf6686b..c2da976533a 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1609,7 +1609,6 @@ out:
         return ret;
 }
 
-
 static int
 __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
 {
@@ -1617,13 +1616,25 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
         struct posix_fd  *pfd = NULL;
         int               ret = -1;
         char             *real_path = NULL;
+        char             *unlink_path = NULL;
         int               _fd = -1;
         DIR              *dir = NULL;
 
+        struct posix_private    *priv      = NULL;
+
+        priv = this->private;
+
         ret = __fd_ctx_get (fd, this, &tmp_pfd);
         if (ret == 0) {
                 pfd = (void *)(long) tmp_pfd;
-                ret = 0;
+                goto out;
+        }
+        if (!fd_is_anonymous(fd)) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        P_MSG_READ_FAILED,
+                        "Failed to get fd context for a non-anonymous fd, "
+                        "file: %s, gfid: %s", real_path,
+                        uuid_utoa (fd->inode->gfid));
                 goto out;
         }
 
@@ -1634,7 +1645,7 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
         MAKE_HANDLE_PATH (real_path, this, fd->inode->gfid, NULL);
         if (!real_path) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
-                        P_MSG_HANDLE_PATH_CREATE_FAILED,
+                        P_MSG_READ_FAILED,
                         "Failed to create handle path (%s)",
                         uuid_utoa (fd->inode->gfid));
                 ret = -1;
@@ -1660,6 +1671,16 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
         if (fd->inode->ia_type == IA_IFREG) {
                 _fd = open (real_path, O_RDWR|O_LARGEFILE);
                 if (_fd == -1) {
+                        POSIX_GET_FILE_UNLINK_PATH (priv->base_path,
+                                                    fd->inode->gfid,
+                                                    unlink_path);
+                        _fd = open (unlink_path, fd->flags);
+                }
+                if (_fd == -1) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_READ_FAILED,
+                                "Failed to get anonymous "
+                                "real_path: %s _fd = %d", real_path, _fd);
                         GF_FREE (pfd);
                         pfd = NULL;
                         goto out;
@@ -2161,3 +2182,40 @@ posix_fdget_objectsignature (int fd, dict_t *xattr)
  error_return:
         return -EINVAL;
 }
+
+
+int
+posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = ctx_int;
+
+out:
+        return ret;
+}
+
+
+int
+posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx)
+{
+        int             ret = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ret = inode_ctx_set (inode, this, &ctx);
+out:
+        return ret;
+}
diff --git a/xlators/storage/posix/src/posix-mem-types.h b/xlators/storage/posix/src/posix-mem-types.h
index 81752c17e78..b463c086be5 100644
--- a/xlators/storage/posix/src/posix-mem-types.h
+++ b/xlators/storage/posix/src/posix-mem-types.h
@@ -21,6 +21,7 @@ enum gf_posix_mem_types_ {
         gf_posix_mt_posix_dev_t,
         gf_posix_mt_trash_path,
 	gf_posix_mt_paiocb,
+        gf_posix_mt_inode_ctx_t,
         gf_posix_mt_end
 };
 #endif
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 600ebf3f0fa..d57d53d5fca 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -106,10 +106,32 @@ int
 posix_forget (xlator_t *this, inode_t *inode)
 {
         uint64_t tmp_cache = 0;
-        if (!inode_ctx_del (inode, this, &tmp_cache))
-                dict_destroy ((dict_t *)(long)tmp_cache);
+        int ret = 0;
+        char *unlink_path = NULL;
+        struct posix_private    *priv_posix = NULL;
 
-        return 0;
+        priv_posix = (struct posix_private *) this->private;
+
+        ret = inode_ctx_del (inode, this, &tmp_cache);
+        if (ret < 0) {
+                ret = 0;
+                goto out;
+        }
+        if (tmp_cache == GF_UNLINK_TRUE) {
+                POSIX_GET_FILE_UNLINK_PATH(priv_posix->base_path,
+                                           inode->gfid, unlink_path);
+                if (!unlink_path) {
+                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                                P_MSG_UNLINK_FAILED,
+                                "Failed to remove gfid :%s",
+                                uuid_utoa (inode->gfid));
+                        ret = -1;
+                        goto out;
+                }
+                ret = sys_unlink(unlink_path);
+        }
+out:
+        return ret;
 }
 
 /* Regular fops */
@@ -1461,16 +1483,93 @@ out:
         return 0;
 }
 
+int
+posix_add_unlink_to_ctx (inode_t *inode, xlator_t *this, char *unlink_path)
+{
+        uint64_t ctx = GF_UNLINK_FALSE;
+        int ret = 0;
+
+        if (!unlink_path) {
+                gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                        P_MSG_UNLINK_FAILED,
+                        "Creation of unlink entry failed for gfid: %s",
+                        unlink_path);
+                ret = -1;
+                goto out;
+        }
+
+        ctx = GF_UNLINK_TRUE;
+        ret = posix_inode_ctx_set (inode, this, ctx);
+        if (ret < 0) {
+                goto out;
+        }
+
+out:
+        return ret;
+}
+
+int32_t
+posix_move_gfid_to_unlink (xlator_t *this, uuid_t gfid, loc_t *loc)
+{
+        char *unlink_path = NULL;
+        char *gfid_path = NULL;
+        struct stat  stbuf = {0, };
+        int ret = 0;
+        struct posix_private    *priv_posix = NULL;
+
+        priv_posix = (struct posix_private *) this->private;
+
+        MAKE_HANDLE_GFID_PATH (gfid_path, this, gfid, NULL);
+
+        POSIX_GET_FILE_UNLINK_PATH (priv_posix->base_path,
+                                    loc->inode->gfid, unlink_path);
+        if (!unlink_path) {
+                ret = -1;
+                goto out;
+        }
+        gf_msg_debug (this->name, 0,
+                      "Moving gfid: %s to unlink_path : %s",
+                      gfid_path, unlink_path);
+        ret = sys_rename (gfid_path, unlink_path);
+        if (ret < 0) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        P_MSG_UNLINK_FAILED,
+                        "Creation of unlink entry failed for gfid: %s",
+                        unlink_path);
+                goto out;
+        }
+        ret = posix_add_unlink_to_ctx (loc->inode, this, unlink_path);
+        if (ret < 0)
+                goto out;
+
+out:
+        return ret;
+}
+
 int32_t
 posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
-                                    struct iatt *stbuf, int32_t *op_errno)
+                                    struct iatt *stbuf, int32_t *op_errno,
+                                    loc_t *loc)
 {
-        int32_t             ret      =   0;
+        int32_t                 ret    = 0;
+        struct posix_private    *priv  = NULL;
+        int fd_count = 0;
 
-        /*  Unlink the gfid_handle_first */
+        priv = this->private;
 
+        /*  Unlink the gfid_handle_first */
         if (stbuf && stbuf->ia_nlink == 1) {
-                ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
+
+                LOCK (&loc->inode->lock);
+
+                if (loc->inode->fd_count == 0) {
+                        UNLOCK (&loc->inode->lock);
+                        ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
+                } else {
+                        UNLOCK (&loc->inode->lock);
+                        ret = posix_move_gfid_to_unlink (this, stbuf->ia_gfid,
+                                                         loc);
+                }
                 if (ret) {
                         gf_msg (this->name, GF_LOG_ERROR, errno,
                                 P_MSG_UNLINK_FAILED, "unlink of gfid handle "
@@ -1484,7 +1583,6 @@ posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
         if (ret == -1) {
                 if (op_errno)
                         *op_errno = errno;
-
                 gf_msg (this->name, GF_LOG_ERROR, errno, P_MSG_UNLINK_FAILED,
                         "unlink of %s failed", real_path);
                 goto err;
@@ -1638,7 +1736,7 @@ posix_unlink (call_frame_t *frame, xlator_t *this,
         }
 
         op_ret =  posix_unlink_gfid_handle_and_entry (this, real_path, &stbuf,
-                                                      &op_errno);
+                                                      &op_errno, loc);
         if (op_ret == -1) {
                 goto out;
         }
@@ -6177,6 +6275,77 @@ out:
 	return ret;
 }
 
+int32_t
+posix_create_unlink_dir (xlator_t *this) {
+
+        char  *unlink_path = NULL;
+        char  *landfill_path = NULL;
+        struct posix_private *priv = NULL;
+        struct stat           stbuf;
+        int ret = -1;
+        uuid_t                gfid = {0};
+        char                  gfid_str[64] = {0};
+
+        priv = this->private;
+
+        unlink_path = alloca (strlen (priv->base_path) + 1 +
+                              strlen (GF_UNLINK_PATH) + 1);
+        sprintf (unlink_path, "%s/%s", priv->base_path,
+                 GF_UNLINK_PATH);
+
+        gf_uuid_generate (gfid);
+        uuid_utoa_r (gfid, gfid_str);
+
+        landfill_path = alloca (strlen (priv->base_path) + 1 +
+                                strlen (GF_LANDFILL_PATH) + 1 +
+                                strlen (gfid_str) + 1);
+        sprintf (landfill_path, "%s/%s/%s", priv->base_path,
+                 GF_LANDFILL_PATH, gfid_str);
+
+        ret = sys_stat (unlink_path, &stbuf);
+        switch (ret) {
+        case -1:
+                if (errno != ENOENT) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_HANDLE_CREATE,
+                                "Checking for %s failed",
+                                unlink_path);
+                        return -1;
+                }
+                break;
+        case 0:
+                if (!S_ISDIR (stbuf.st_mode)) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                P_MSG_HANDLE_CREATE,
+                                "Not a directory: %s",
+                                unlink_path);
+                        return -1;
+                }
+                ret = sys_rename (unlink_path, landfill_path);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                P_MSG_HANDLE_CREATE,
+                                "Can not delete directory %s ",
+                                unlink_path);
+                        return -1;
+                }
+                break;
+        default:
+                break;
+        }
+        ret = sys_mkdir (unlink_path, 0600);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        P_MSG_HANDLE_CREATE,
+                        "Creating directory %s failed",
+                        unlink_path);
+                return -1;
+        }
+
+        return 0;
+}
+
+
 
 /**
  * init -
@@ -6588,6 +6757,15 @@ init (xlator_t *this)
                 goto out;
 	}
 
+        op_ret = posix_create_unlink_dir (this);
+        if (op_ret == -1) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        P_MSG_HANDLE_CREATE,
+                        "Creation of unlink directory failed");
+                ret = -1;
+                goto out;
+        }
+
 	_private->aio_init_done = _gf_false;
 	_private->aio_capable = _gf_false;
 
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index f8230be324b..9ed82fc4bbc 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -61,6 +61,8 @@
                                                + SLEN(GF_HIDDEN_PATH) + SLEN("/") \
                                                + SLEN("00/")            \
                                                + SLEN("00/") + SLEN(UUID0_STR) + 1) /* '\0' */;
+#define GF_UNLINK_TRUE 0x0000000000000001
+#define GF_UNLINK_FALSE 0x0000000000000000
 
 /**
  * posix_fd - internal structure common to file and directory fd's
@@ -189,14 +191,36 @@ typedef struct {
         int32_t     op_errno;
 } posix_xattr_filler_t;
 
-
 #define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
 
 #define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)
 
 #define POSIX_PATH_MAX(this) (((struct posix_private *)this->private)->path_max)
 
+#define POSIX_GET_FILE_UNLINK_PATH(base_path, gfid, unlink_path)                           \
+        do {                                                                               \
+                int  path_len = 0;                                                         \
+                char gfid_str[64] = {0};                                                   \
+                uuid_utoa_r (gfid, gfid_str);                                              \
+                path_len = strlen (base_path) + 1 +                                        \
+                          strlen (GF_UNLINK_PATH) + 1 +                                    \
+                          strlen (gfid_str) + 1;                                           \
+                unlink_path = alloca (path_len);                                           \
+                if (!unlink_path) {                                                        \
+                        gf_msg ("posix", GF_LOG_ERROR, ENOMEM,                             \
+                                P_MSG_UNLINK_FAILED,                                       \
+                                "Failed to get unlink_path");                              \
+                        break;                                                             \
+                }                                                                          \
+                sprintf (unlink_path, "%s/%s/%s",                                          \
+                         base_path, GF_UNLINK_PATH, gfid_str);                             \
+         } while (0)
+
+
 /* Helper functions */
+int posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx);
+int posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx);
+
 int posix_gfid_set (xlator_t *this, const char *path, loc_t *loc,
                     dict_t *xattr_req);
 int posix_fdstat (xlator_t *this, int fd, struct iatt *stbuf_p);