From 14555df7c6bc74622eb2283ded1c1f10980180ce Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Wed, 10 Jan 2018 15:01:59 +0530 Subject: gfapi : New APIs have been added to use lease feature in gluster Following APIs glfs_h_lease(), glfs_lease() added, so that gfapi applications can set and get lease which enables more efficient client side caching. Updates: #350 Change-Id: Iede85be9af1d4df969b890d0937ed0afa4ca6596 Signed-off-by: Poornima G Signed-off-by: Soumya Koduri Signed-off-by: Jiffin Tony Thottan --- api/src/gfapi-messages.h | 4 +- api/src/gfapi.aliases | 2 + api/src/gfapi.map | 2 + api/src/glfs-fops.c | 135 ++++++++++++++++++++++++++++++++++---- api/src/glfs-handleops.c | 68 +++++++++++++++++-- api/src/glfs-handles.h | 4 ++ api/src/glfs-internal.h | 13 ++-- api/src/glfs-resolve.c | 1 + api/src/glfs.c | 2 +- api/src/glfs.h | 77 +++++++++++++++++++++- libglusterfs/src/libglusterfs.sym | 1 + 11 files changed, 280 insertions(+), 29 deletions(-) diff --git a/api/src/gfapi-messages.h b/api/src/gfapi-messages.h index f231ce99a47..4d9dff7196d 100644 --- a/api/src/gfapi-messages.h +++ b/api/src/gfapi-messages.h @@ -75,7 +75,9 @@ GLFS_MSGID(API, API_MSG_STATEDUMP_FAILED, API_MSG_XREADDIRP_R_FAILED, API_MSG_LOCK_INSERT_MERGE_FAILED, - API_MSG_SETTING_LOCK_TYPE_FAILED + API_MSG_SETTING_LOCK_TYPE_FAILED, + API_MSG_INODE_FIND_FAILED, + API_MSG_FDCTX_SET_FAILED ); #endif /* !_GFAPI_MESSAGES_H__ */ diff --git a/api/src/gfapi.aliases b/api/src/gfapi.aliases index 88d361dc329..0cb2e5c1baf 100644 --- a/api/src/gfapi.aliases +++ b/api/src/gfapi.aliases @@ -170,3 +170,5 @@ _pub_glfs_upcall_unregister _glfs_upcall_unregister$GFAPI_3.13.0 _pub_glfs_setfsleaseid _glfs_setfsleaseid$GFAPI_4.0.0 _pub_glfs_file_lock _glfs_file_lock$GFAPI_4.0.0 +_pub_glfs_lease _glfs_lease$GFAPI_4.0.0 +_pub_glfs_h_lease _glfs_h_lease$GFAPI_4.0.0 diff --git a/api/src/gfapi.map b/api/src/gfapi.map index fc47a3b8f42..4dc20e33f4d 100644 --- a/api/src/gfapi.map +++ b/api/src/gfapi.map @@ -225,4 +225,6 @@ GFAPI_4.0.0 { global: glfs_setfsleaseid; glfs_file_lock; + glfs_lease; + glfs_h_lease; } GFAPI_3.13.0; diff --git a/api/src/glfs-fops.c b/api/src/glfs-fops.c index 833ff336634..bb51f729562 100644 --- a/api/src/glfs-fops.c +++ b/api/src/glfs-fops.c @@ -39,7 +39,11 @@ int glfs_mark_glfd_for_deletion (struct glfs_fd *glfd) { - glfd->state = GLFD_CLOSE; + LOCK (&glfd->lock); + { + glfd->state = GLFD_CLOSE; + } + UNLOCK (&glfd->lock); GF_REF_PUT (glfd); @@ -56,10 +60,31 @@ glfs_mark_glfd_for_deletion (struct glfs_fd *glfd) gf_boolean_t glfs_is_glfd_still_valid (struct glfs_fd *glfd) { - if (glfd->state != GLFD_CLOSE) - return _gf_true; + gf_boolean_t ret = _gf_false; + + LOCK (&glfd->lock); + { + if (glfd->state != GLFD_CLOSE) + ret = _gf_true; + } + UNLOCK (&glfd->lock); - return _gf_false; + return ret; +} + +void +glfd_set_state_bind (struct glfs_fd *glfd) +{ + LOCK (&glfd->lock); + { + glfd->state = GLFD_OPEN; + } + UNLOCK (&glfd->lock); + + fd_bind (glfd->fd); + glfs_fd_bind (glfd); + + return; } /* @@ -227,9 +252,7 @@ out: GF_REF_PUT (glfd); glfd = NULL; } else if (glfd) { - glfd->state = GLFD_OPEN; - fd_bind (glfd->fd); - glfs_fd_bind (glfd); + glfd_set_state_bind (glfd); } glfs_subvol_done (fs, subvol); @@ -549,9 +572,7 @@ out: GF_REF_PUT (glfd); glfd = NULL; } else if (glfd) { - glfd->state = GLFD_OPEN; - fd_bind (glfd->fd); - glfs_fd_bind (glfd); + glfd_set_state_bind (glfd); } glfs_subvol_done (fs, subvol); @@ -2411,9 +2432,7 @@ out: GF_REF_PUT (glfd); glfd = NULL; } else if (glfd) { - glfd->state = GLFD_OPEN; - fd_bind (glfd->fd); - glfs_fd_bind (glfd); + glfd_set_state_bind (glfd); } glfs_subvol_done (fs, subvol); @@ -4636,6 +4655,12 @@ priv_glfs_process_upcall_event (struct glfs *fs, void *data) gf_msg_trace (THIS->name, 0, "Upcall gfapi gfid = %s" , (char *)(upcall_data->gfid)); + /* * + * TODO: RECALL LEASE + * Refer issue #350 + * Proposed patch https://review.gluster.org/#/c/14019/ + * */ + if (fs->up_cbk) { /* upcall cbk registered */ (void) glfs_cbk_upcall_data (fs, upcall_data); } else { @@ -4972,3 +4997,87 @@ out: } GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_stat, 3.11.0); +void +gf_lease_to_glfs_lease (struct gf_lease *gf_lease, struct glfs_lease *lease) +{ + lease->cmd = gf_lease->cmd; + lease->lease_type = gf_lease->lease_type; + memcpy (lease->lease_id, gf_lease->lease_id, LEASE_ID_SIZE); +} + +void +glfs_lease_to_gf_lease (struct glfs_lease *lease, struct gf_lease *gf_lease) +{ + gf_lease->cmd = lease->cmd; + gf_lease->lease_type = lease->lease_type; + memcpy (gf_lease->lease_id, lease->lease_id, LEASE_ID_SIZE); +} + +int +pub_glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease) +{ + int ret = -1; + loc_t loc = {0, }; + xlator_t *subvol = NULL; + fd_t *fd = NULL; + struct gf_lease gf_lease = {0, }; + + DECLARE_OLD_THIS; + __GLFS_ENTRY_VALIDATE_FD (glfd, invalid_fs); + + GF_REF_GET (glfd); + + if (!is_valid_lease_id (lease->lease_id)) { + ret = -1; + errno = EINVAL; + goto out; + } + + subvol = glfs_active_subvol (glfd->fs); + if (!subvol) { + ret = -1; + errno = EIO; + goto out; + } + + fd = glfs_resolve_fd (glfd->fs, subvol, glfd); + if (!fd) { + ret = -1; + errno = EBADFD; + goto out; + } + + /* populate loc */ + GLFS_LOC_FILL_INODE (fd->inode, loc, out); + + glfs_lease_to_gf_lease (lease, &gf_lease); + + ret = syncop_lease (subvol, &loc, &gf_lease, NULL, NULL); + DECODE_SYNCOP_ERR (ret); + + gf_lease_to_glfs_lease (&gf_lease, lease); + + /* TODO: Add leases for client replay + if (ret == 0 && (cmd == F_SETLK || cmd == F_SETLKW)) + fd_lk_insert_and_merge (fd, cmd, &saved_flock); + */ + +out: + if (fd) + fd_unref (fd); + + if (glfd) { + GF_REF_PUT (glfd); + glfd = NULL; + } + + if (subvol) + glfs_subvol_done (glfd->fs, subvol); + + __GLFS_EXIT_FS; + +invalid_fs: + return ret; +} + +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_lease, 4.0.0); diff --git a/api/src/glfs-handleops.c b/api/src/glfs-handleops.c index 38a9a7ad2a5..a500aae3aee 100644 --- a/api/src/glfs-handleops.c +++ b/api/src/glfs-handleops.c @@ -682,8 +682,6 @@ pub_glfs_h_open (struct glfs *fs, struct glfs_object *object, int flags) DECODE_SYNCOP_ERR (ret); glfd->fd->flags = flags; - fd_bind (glfd->fd); - glfs_fd_bind (glfd); out: loc_wipe (&loc); @@ -695,7 +693,7 @@ out: GF_REF_PUT (glfd); glfd = NULL; } else if (glfd) { - glfd->state = GLFD_OPEN; + glfd_set_state_bind (glfd); } glfs_subvol_done (fs, subvol); @@ -1150,9 +1148,7 @@ out: GF_REF_PUT (glfd); glfd = NULL; } else if (glfd) { - glfd->state = GLFD_OPEN; - fd_bind (glfd->fd); - glfs_fd_bind (glfd); + glfd_set_state_bind (glfd); } glfs_subvol_done (fs, subvol); @@ -2460,3 +2456,63 @@ out: return NULL; } GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_xreaddirplus_get_object, 3.11.0); + +int +pub_glfs_h_lease (struct glfs *fs, struct glfs_object *object, + struct glfs_lease *lease) +{ + int ret = -1; + xlator_t *subvol = NULL; + inode_t *inode = NULL; + loc_t loc = {0, }; + struct gf_lease gf_lease = {0, }; + + /* validate in args */ + if ((fs == NULL) || (object == NULL)) { + errno = EINVAL; + return -1; + } + + DECLARE_OLD_THIS; + __GLFS_ENTRY_VALIDATE_FS (fs, invalid_fs); + + /* get the active volume */ + subvol = glfs_active_subvol (fs); + if (!subvol) { + ret = -1; + errno = EIO; + goto out; + } + + /* get/refresh the in arg objects inode in correlation to the xlator */ + inode = glfs_resolve_inode (fs, subvol, object); + if (!inode) { + errno = ESTALE; + goto out; + } + + /* populate loc */ + GLFS_LOC_FILL_INODE (inode, loc, out); + + glfs_lease_to_gf_lease (lease, &gf_lease); + + ret = syncop_lease (subvol, &loc, &gf_lease, NULL, NULL); + DECODE_SYNCOP_ERR (ret); + + gf_lease_to_glfs_lease (&gf_lease, lease); + +out: + loc_wipe (&loc); + + if (inode) + inode_unref (inode); + + glfs_subvol_done (fs, subvol); + + __GLFS_EXIT_FS; + +invalid_fs: + return ret; +} + +GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_h_lease, 4.0.0); diff --git a/api/src/glfs-handles.h b/api/src/glfs-handles.h index 211595933fc..97c058a11c7 100644 --- a/api/src/glfs-handles.h +++ b/api/src/glfs-handles.h @@ -328,6 +328,10 @@ struct glfs_object* glfs_object_copy (struct glfs_object *src); GFAPI_PUBLIC(glfs_object_copy, 3.11.0); +int glfs_h_lease (struct glfs *fs, struct glfs_object *object, + struct glfs_lease *lease) __THROW + GFAPI_PUBLIC(glfs_h_lease, 4.0.0); + __END_DECLS #endif /* !_GLFS_HANDLES_H */ diff --git a/api/src/glfs-internal.h b/api/src/glfs-internal.h index 76b0e34f1bd..145509c3caf 100644 --- a/api/src/glfs-internal.h +++ b/api/src/glfs-internal.h @@ -222,6 +222,8 @@ struct glfs_fd { gf_dirent_t *next; struct dirent *readdirbuf; gf_lkowner_t lk_owner; + glfs_leaseid_t lease_id; /* Stores lease_id of client in glfd */ + gf_lock_t lock; /* lock taken before updating fd state */ }; /* glfs object handle introduced for the alternate gfapi implementation based @@ -460,6 +462,7 @@ glfs_unlock (struct glfs *fs) struct glfs_fd *glfs_fd_new (struct glfs *fs); void glfs_fd_bind (struct glfs_fd *glfd); +void glfd_set_state_bind (struct glfs_fd *glfd); xlator_t *glfs_active_subvol (struct glfs *fs) GFAPI_PRIVATE(glfs_active_subvol, 3.4.0); @@ -604,11 +607,11 @@ glfd_entry_next (struct glfs_fd *glfd, int plus); void gf_dirent_to_dirent (gf_dirent_t *gf_dirent, struct dirent *dirent); -/* - * Nobody needs this call at all yet except for the test script. - */ -int glfs_ipc (glfs_fd_t *fd, int cmd, void *xd_in, void **xd_out) __THROW - GFAPI_PRIVATE(glfs_ipc, 3.12.0); +void +gf_lease_to_glfs_lease (struct gf_lease *gf_lease, struct glfs_lease *lease); + +void +glfs_lease_to_gf_lease (struct glfs_lease *lease, struct gf_lease *gf_lease); void glfs_release_upcall (void *ptr); #endif /* !_GLFS_INTERNAL_H */ diff --git a/api/src/glfs-resolve.c b/api/src/glfs-resolve.c index 16093ec2215..6c9fc38901f 100644 --- a/api/src/glfs-resolve.c +++ b/api/src/glfs-resolve.c @@ -953,6 +953,7 @@ __glfs_active_subvol (struct glfs *fs) } __glfs_migrate_openfds (fs, new_subvol); + /* TODO: Migrate the fds and inodes which have leases to the new graph (issue #350)*/ /* switching @active_subvol and @cwd should be atomic diff --git a/api/src/glfs.c b/api/src/glfs.c index 8c6916d0d85..2a7ae2f3986 100644 --- a/api/src/glfs.c +++ b/api/src/glfs.c @@ -597,7 +597,7 @@ pub_glfs_setfsgroups (size_t size, const gid_t *list) GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_setfsgroups, 3.4.2); int -pub_glfs_setfsleaseid (leaseid_t leaseid) +pub_glfs_setfsleaseid (glfs_leaseid_t leaseid) { int ret = -1; char *gleaseid = NULL; diff --git a/api/src/glfs.h b/api/src/glfs.h index 5ffed1e0853..4613cd193ea 100644 --- a/api/src/glfs.h +++ b/api/src/glfs.h @@ -370,8 +370,8 @@ int glfs_get_volumeid (struct glfs *fs, char *volid, size_t size) __THROW struct glfs_fd; typedef struct glfs_fd glfs_fd_t; -#define GFAPI_LEASE_ID_SIZE 16 /* 128bits */ -typedef char leaseid_t[GFAPI_LEASE_ID_SIZE]; +#define GLFS_LEASE_ID_SIZE 16 /* 128bits */ +typedef char glfs_leaseid_t[GLFS_LEASE_ID_SIZE]; /* * PER THREAD IDENTITY MODIFIERS @@ -404,7 +404,7 @@ int glfs_setfsgid (gid_t fsgid) __THROW GFAPI_PUBLIC(glfs_setfsgid, 3.4.2); int glfs_setfsgroups (size_t size, const gid_t *list) __THROW GFAPI_PUBLIC(glfs_setfsgroups, 3.4.2); -int glfs_setfsleaseid (leaseid_t leaseid) __THROW +int glfs_setfsleaseid (glfs_leaseid_t leaseid) __THROW GFAPI_PUBLIC(glfs_setfsleaseid, 4.0.0); /* @@ -1083,5 +1083,76 @@ int glfs_upcall_unregister (struct glfs *fs, uint32_t event_list); GFAPI_PUBLIC(glfs_upcall_unregister, 3.13.0); +/* Lease Types */ +enum glfs_lease_types { + GLFS_RD_LEASE = 1, + GLFS_RW_LEASE = 2, +}; +typedef enum glfs_lease_types glfs_lease_types_t; + +/* Lease cmds */ +enum glfs_lease_cmds { + GLFS_GET_LEASE = 1, + GLFS_SET_LEASE = 2, + GLFS_UNLK_LEASE = 3, +}; +typedef enum glfs_lease_cmds glfs_lease_cmds_t; + +struct glfs_lease { + glfs_lease_cmds_t cmd; + glfs_lease_types_t lease_type; + glfs_leaseid_t lease_id; + unsigned int lease_flags; +}; + +typedef void (*glfs_recall_cbk) (struct glfs_lease lease, void *data); + +/* + SYNOPSIS + + glfs_lease: Takes a lease on a file. + + DESCRIPTION + + This function takes lease on an open file. + + PARAMETERS + + @glfd: The fd of the file on which lease should be taken, + this fd is returned by glfs_open/glfs_create. + + @lease: Struct that defines the lease operation to be performed + on the file. + @lease.cmd - Can be one of the following values + GF_GET_LEASE: Get the lease type currently present on the file, + lease.lease_type will contain GF_RD_LEASE + or GF_RW_LEASE or 0 if no leases. + GF_SET_LEASE: Set the lease of given lease.lease_type on the file. + GF_UNLK_LEASE: Unlock the lease present on the given fd. + Note that the every lease request should have + a corresponding unlk_lease. + + @lease.lease_type - Can be one of the following values + GF_RD_LEASE: Read lease on a file, shared lease. + GF_RW_LEASE: Read-Write lease on a file, exclusive lease. + + @lease.lease_id - A unique identification of lease, 128bits. + + @fn: This is the function that is invoked when the lease has to be recalled + @data: It is a cookie, this pointer is returned as a part of recall + + fn and data field are stored as a part of glfs_fd, hence if there are multiple + glfs_lease calls, each of them updates the fn and data fileds. glfs_recall_cbk + will be invoked with the last updated fn and data + + RETURN VALUES + 0: Successfull completion + <0: Failure. @errno will be set with the type of failure +*/ + +int glfs_lease (struct glfs_fd *glfd, struct glfs_lease *lease, + glfs_recall_cbk fn, void *data) __THROW + GFAPI_PUBLIC(glfs_lease, 4.0.0); + __END_DECLS #endif /* !_GLFS_H */ diff --git a/libglusterfs/src/libglusterfs.sym b/libglusterfs/src/libglusterfs.sym index a515caec3e9..500f69a9893 100644 --- a/libglusterfs/src/libglusterfs.sym +++ b/libglusterfs/src/libglusterfs.sym @@ -965,6 +965,7 @@ syncop_write syncop_writev syncop_xattrop syncop_zerofill +syncop_lease synctask_get synctask_new synctask_new1 -- cgit