1 files changed, 172 insertions, 9 deletions

diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c
index ce0cebc20c1..fdffc7b7cb3 100644
--- a/api/src/glfs-fops.c
+++ b/api/src/glfs-fops.c
@@ -28,6 +28,35 @@
 #define READDIRBUF_SIZE (sizeof(struct dirent) + GF_NAME_MAX + 1)
 
 /*
+ * This function will mark glfd for deletion and decrement its refcount.
+ */
+int
+glfs_mark_glfd_for_deletion (struct glfs_fd *glfd)
+{
+        glfd->state = GLFD_CLOSE;
+
+        GF_REF_PUT (glfd);
+
+        return 0;
+}
+
+/* This function is usefull for all async fops. There is chance that glfd is
+ * closed before async fop is completed. When glfd is closed we change the
+ * state to GLFD_CLOSE.
+ *
+ * This function will return _gf_true if the glfd is still valid else return
+ * _gf_false.
+ */
+gf_boolean_t
+glfs_is_glfd_still_valid (struct glfs_fd *glfd)
+{
+        if (glfd->state != GLFD_CLOSE)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+/*
  * This routine is called when an upcall event of type
  * 'GF_UPCALL_CACHE_INVALIDATION' is received.
  * It makes a copy of the contents of the upcall cache-invalidation
@@ -190,9 +219,10 @@ out:
 	loc_wipe (&loc);
 
 	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
+                GF_REF_PUT (glfd);
 		glfd = NULL;
 	} else if (glfd) {
+                glfd->state = GLFD_OPEN;
 		fd_bind (glfd->fd);
 		glfs_fd_bind (glfd);
 	}
@@ -207,7 +237,6 @@ invalid_fs:
 
 GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_open, 3.4.0);
 
-
 int
 pub_glfs_close (struct glfs_fd *glfd)
 {
@@ -237,11 +266,11 @@ pub_glfs_close (struct glfs_fd *glfd)
         DECODE_SYNCOP_ERR (ret);
 out:
 	fs = glfd->fs;
-	glfs_fd_destroy (glfd);
 
-	if (fd)
-		fd_unref (fd);
+        if (fd)
+                fd_unref (fd);
 
+        glfs_mark_glfd_for_deletion (glfd);
 	glfs_subvol_done (fs, subvol);
 
         __GLFS_EXIT_FS;
@@ -342,6 +371,8 @@ pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -364,6 +395,8 @@ pub_glfs_fstat (struct glfs_fd *glfd, struct stat *stat)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -502,9 +535,10 @@ out:
 		dict_unref (xattr_req);
 
 	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
+                GF_REF_PUT (glfd);
 		glfd = NULL;
 	} else if (glfd) {
+                glfd->state = GLFD_OPEN;
 		fd_bind (glfd->fd);
 		glfs_fd_bind (glfd);
 	}
@@ -530,6 +564,8 @@ pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	switch (whence) {
 	case SEEK_SET:
 		glfd->offset = offset;
@@ -551,6 +587,9 @@ pub_glfs_lseek (struct glfs_fd *glfd, off_t offset, int whence)
                 errno = EINVAL;
 	}
 
+        if (glfd)
+                GF_REF_PUT (glfd);
+
         __GLFS_EXIT_FS;
 
         if (ret != -1)
@@ -580,6 +619,8 @@ pub_glfs_preadv (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -615,6 +656,8 @@ out:
 
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -693,7 +736,18 @@ glfs_io_async_cbk (int ret, call_frame_t *frame, void *data)
 {
 	struct glfs_io  *gio = data;
 
-	gio->fn (gio->glfd, ret, gio->data);
+        /* If the fd is already closed then
+         * no need to do the callback */
+        if (glfs_is_glfd_still_valid (gio->glfd)) {
+                gio->fn (gio->glfd, ret, gio->data);
+        }
+
+        /* Since the async operation is complete
+         * release the ref taken during the start
+         * of async operation
+         */
+        if (gio->glfd)
+                GF_REF_PUT (gio->glfd);
 
 	GF_FREE (gio->iov);
 	GF_FREE (gio);
@@ -769,6 +823,9 @@ glfs_preadv_async_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 	glfd = gio->glfd;
 	fs = glfd->fs;
 
+        if (!glfs_is_glfd_still_valid (glfd))
+                goto err;
+
 	if (op_ret <= 0)
 		goto out;
 
@@ -779,6 +836,13 @@ out:
 	errno = op_errno;
 	gio->fn (gio->glfd, op_ret, gio->data);
 
+err:
+        /* Since the async operation is complete
+         * release the ref taken during the start
+         * of async operation
+         */
+        GF_REF_PUT (glfd);
+
 	GF_FREE (gio->iov);
 	GF_FREE (gio);
 	STACK_DESTROY (frame->root);
@@ -803,6 +867,8 @@ pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -856,6 +922,8 @@ pub_glfs_preadv_async (struct glfs_fd *glfd, const struct iovec *iovec,
 
 out:
         if (ret) {
+                if (glfd)
+                        GF_REF_PUT (glfd);
                 if (gio) {
                         GF_FREE (gio->iov);
                         GF_FREE (gio);
@@ -945,6 +1013,8 @@ pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -1005,6 +1075,8 @@ pub_glfs_pwritev (struct glfs_fd *glfd, const struct iovec *iovec, int iovcnt,
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1100,11 +1172,17 @@ pub_glfs_pwritev_async (struct glfs_fd *glfd, const struct iovec *iovec,
 	gio->fn     = fn;
 	gio->data   = data;
 
+        /* Need to take explicit ref so that the fd
+         * is not destroyed before the fop is complete
+         */
+        GF_REF_GET (glfd);
+
 	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
 			    glfs_io_async_task, glfs_io_async_cbk,
 			    NULL, gio);
 
 	if (ret) {
+                GF_REF_PUT (glfd);
 		GF_FREE (gio->iov);
 		GF_FREE (gio);
 	}
@@ -1179,6 +1257,8 @@ pub_glfs_fsync (struct glfs_fd *glfd)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -1198,6 +1278,8 @@ pub_glfs_fsync (struct glfs_fd *glfd)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1223,6 +1305,11 @@ glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
 		return -1;
 	}
 
+        /* Need to take explicit ref so that the fd
+         * is not destroyed before the fop is complete
+         */
+        GF_REF_GET (glfd);
+
 	gio->op     = GF_FOP_FSYNC;
 	gio->glfd   = glfd;
 	gio->flags  = dataonly;
@@ -1234,6 +1321,7 @@ glfs_fsync_async_common (struct glfs_fd *glfd, glfs_io_cbk fn, void *data,
 			    NULL, gio);
 
 	if (ret) {
+                GF_REF_PUT (glfd);
 		GF_FREE (gio->iov);
 		GF_FREE (gio);
 	}
@@ -1272,6 +1360,8 @@ pub_glfs_fdatasync (struct glfs_fd *glfd)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -1291,6 +1381,8 @@ pub_glfs_fdatasync (struct glfs_fd *glfd)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1332,6 +1424,8 @@ pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -1351,6 +1445,8 @@ pub_glfs_ftruncate (struct glfs_fd *glfd, off_t offset)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -1385,11 +1481,17 @@ pub_glfs_ftruncate_async (struct glfs_fd *glfd, off_t offset, glfs_io_cbk fn,
 	gio->fn     = fn;
 	gio->data   = data;
 
+        /* Need to take explicit ref so that the fd
+         * is not destroyed before the fop is complete
+         */
+        GF_REF_GET (glfd);
+
 	ret = synctask_new (pub_glfs_from_glfd (glfd)->ctx->env,
 			    glfs_io_async_task, glfs_io_async_cbk,
 			    NULL, gio);
 
 	if (ret) {
+                GF_REF_PUT (glfd);
 		GF_FREE (gio->iov);
 		GF_FREE (gio);
 	}
@@ -2095,9 +2197,10 @@ out:
 	loc_wipe (&loc);
 
 	if (ret && glfd) {
-		glfs_fd_destroy (glfd);
+		GF_REF_PUT (glfd);
 		glfd = NULL;
 	} else if (glfd) {
+                glfd->state = GLFD_OPEN;
 		fd_bind (glfd->fd);
 		glfs_fd_bind (glfd);
 	}
@@ -2123,7 +2226,7 @@ pub_glfs_closedir (struct glfs_fd *glfd)
 
 	gf_dirent_free (list_entry (&glfd->entries, gf_dirent_t, list));
 
-	glfs_fd_destroy (glfd);
+        glfs_mark_glfd_for_deletion (glfd);
 
         __GLFS_EXIT_FS;
 
@@ -2185,6 +2288,11 @@ pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        /* Need to take explicit ref so that the fd
+         * is not destroyed before the fop is complete
+         */
+        GF_REF_GET (glfd);
+
 	gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
 	if (!gio) {
 		errno = ENOMEM;
@@ -2203,6 +2311,7 @@ pub_glfs_discard_async (struct glfs_fd *glfd, off_t offset, size_t len,
 			    NULL, gio);
 
 	if (ret) {
+                GF_REF_PUT (glfd);
 		GF_FREE (gio->iov);
 		GF_FREE (gio);
 	}
@@ -2227,6 +2336,11 @@ pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        /* Need to take explicit ref so that the fd
+         * is not destroyed before the fop is complete
+         */
+        GF_REF_GET (glfd);
+
         gio = GF_CALLOC (1, sizeof (*gio), glfs_mt_glfs_io_t);
         if (!gio) {
                 errno = ENOMEM;
@@ -2245,6 +2359,7 @@ pub_glfs_zerofill_async (struct glfs_fd *glfd, off_t offset, off_t len,
                             NULL, gio);
 
         if (ret) {
+                GF_REF_PUT (glfd);
                 GF_FREE (gio->iov);
                 GF_FREE (gio);
         }
@@ -2424,6 +2539,8 @@ pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	errno = 0;
 
 	if (ext)
@@ -2455,6 +2572,9 @@ pub_glfs_readdirplus_r (struct glfs_fd *glfd, struct stat *stat,
 	}
 
 out:
+        if (glfd)
+                GF_REF_PUT (glfd);
+
         __GLFS_EXIT_FS;
 
 	return ret;
@@ -2601,6 +2721,8 @@ glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -2620,6 +2742,8 @@ glfs_fsetattr (struct glfs_fd *glfd, struct iatt *iatt, int valid)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -2939,6 +3063,8 @@ pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
         if (!name || *name == '\0') {
                 ret = -1;
                 errno = EINVAL;
@@ -2974,6 +3100,8 @@ pub_glfs_fgetxattr (struct glfs_fd *glfd, const char *name, void *value,
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3096,6 +3224,8 @@ pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3119,6 +3249,8 @@ pub_glfs_flistxattr (struct glfs_fd *glfd, void *value, size_t size)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3232,6 +3364,8 @@ pub_glfs_fsetxattr (struct glfs_fd *glfd, const char *name, const void *value,
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
         if (!name || *name == '\0') {
                 ret = -1;
                 errno = EINVAL;
@@ -3273,6 +3407,8 @@ out:
 
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3360,6 +3496,8 @@ pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3379,6 +3517,8 @@ pub_glfs_fremovexattr (struct glfs_fd *glfd, const char *name)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3401,6 +3541,8 @@ pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t le
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3420,6 +3562,8 @@ pub_glfs_fallocate (struct glfs_fd *glfd, int keep_size, off_t offset, size_t le
 out:
 	if (fd)
 		fd_unref(fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3442,6 +3586,8 @@ pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3461,6 +3607,8 @@ pub_glfs_discard (struct glfs_fd *glfd, off_t offset, size_t len)
 out:
 	if (fd)
 		fd_unref(fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3483,6 +3631,8 @@ pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
         DECLARE_OLD_THIS;
         __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
         subvol = glfs_active_subvol (glfd->fs);
         if (!subvol) {
                 errno = EIO;
@@ -3500,6 +3650,8 @@ pub_glfs_zerofill (struct glfs_fd *glfd, off_t offset, off_t len)
 out:
         if (fd)
                 fd_unref(fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
         glfs_subvol_done (glfd->fs, subvol);
 
@@ -3571,6 +3723,8 @@ pub_glfs_fchdir (struct glfs_fd *glfd)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3598,6 +3752,8 @@ pub_glfs_fchdir (struct glfs_fd *glfd)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3759,6 +3915,7 @@ pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
 	subvol = glfs_active_subvol (glfd->fs);
 	if (!subvol) {
 		ret = -1;
@@ -3784,6 +3941,8 @@ pub_glfs_posix_lock (struct glfs_fd *glfd, int cmd, struct flock *flock)
 out:
 	if (fd)
 		fd_unref (fd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (glfd->fs, subvol);
 
@@ -3807,6 +3966,8 @@ pub_glfs_dup (struct glfs_fd *glfd)
         DECLARE_OLD_THIS;
 	__GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs);
 
+        GF_REF_GET (glfd);
+
 	fs = glfd->fs;
 	subvol = glfs_active_subvol (fs);
 	if (!subvol) {
@@ -3832,6 +3993,8 @@ out:
 		fd_unref (fd);
 	if (dupfd)
 		glfs_fd_bind (dupfd);
+        if (glfd)
+                GF_REF_PUT (glfd);
 
 	glfs_subvol_done (fs, subvol);