From 304640e55c0f3c6d15f4e230dc6376e4f5020fea Mon Sep 17 00:00:00 2001 From: Kotresh HR Date: Mon, 29 Jul 2019 18:30:42 +0530 Subject: ctime/rebalance: Heal ctime xattr on directory during rebalance After add-brick and rebalance, the ctime xattr is not present on rebalanced directories on new brick. This patch fixes the same. Note that ctime still doesn't support consistent time across distribute sub-volume. This patch also fixes the in-memory inconsistency of time attributes when metadata is self healed. Change-Id: Ia20506f1839021bf61d4753191e7dc34b31bb2df fixes: bz#1734026 Signed-off-by: Kotresh HR --- xlators/storage/posix/src/posix-entry-ops.c | 8 ++-- xlators/storage/posix/src/posix-helpers.c | 31 ++++++++++-- xlators/storage/posix/src/posix-inode-fd-ops.c | 57 +++++++++------------- xlators/storage/posix/src/posix-metadata.c | 65 ++++++++++++++++++++++++-- xlators/storage/posix/src/posix-metadata.h | 7 +++ xlators/storage/posix/src/posix.h | 5 +- 6 files changed, 125 insertions(+), 48 deletions(-) (limited to 'xlators/storage/posix') diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c index 3c070de3929..30b80f6e4a0 100644 --- a/xlators/storage/posix/src/posix-entry-ops.c +++ b/xlators/storage/posix/src/posix-entry-ops.c @@ -494,7 +494,7 @@ post_op: posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name); } - op_ret = posix_entry_create_xattr_set(this, real_path, xdata); + op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata); if (op_ret) { if (errno != EEXIST) gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED, @@ -822,7 +822,7 @@ posix_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, "setting ACLs on %s failed ", real_path); } - op_ret = posix_entry_create_xattr_set(this, real_path, xdata); + op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata); if (op_ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED, "setting xattrs on %s failed", real_path); @@ -1523,7 +1523,7 @@ posix_symlink(call_frame_t *frame, xlator_t *this, const char *linkname, } ignore: - op_ret = posix_entry_create_xattr_set(this, real_path, xdata); + op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata); if (op_ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED, "setting xattrs on %s failed ", real_path); @@ -2161,7 +2161,7 @@ posix_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name); } ignore: - op_ret = posix_entry_create_xattr_set(this, real_path, xdata); + op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata); if (op_ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED, "setting xattrs on %s failed ", real_path); diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c index c185b4d9049..6c5449083f3 100644 --- a/xlators/storage/posix/src/posix-helpers.c +++ b/xlators/storage/posix/src/posix-helpers.c @@ -1182,11 +1182,15 @@ posix_dump_buffer(xlator_t *this, const char *real_path, const char *key, #endif int -posix_handle_pair(xlator_t *this, const char *real_path, char *key, +posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key, data_t *value, int flags, struct iatt *stbuf) { int sys_ret = -1; int ret = 0; + int op_errno = 0; + struct mdata_iatt mdata_iatt = { + 0, + }; #ifdef GF_DARWIN_HOST_OS const int error_code = EINVAL; #else @@ -1211,6 +1215,23 @@ posix_handle_pair(xlator_t *this, const char *real_path, char *key, /* ignore this key value pair */ ret = 0; goto out; + } else if (!strncmp(key, GF_XATTR_MDATA_KEY, strlen(key))) { + /* This is either by rebalance or self heal. Create the xattr if it's + * not present. Compare and update the larger value if the xattr is + * already present. + */ + if (loc == NULL) { + ret = -EINVAL; + goto out; + } + posix_mdata_iatt_from_disk(&mdata_iatt, + (posix_mdata_disk_t *)value->data); + ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path, + &mdata_iatt, &op_errno); + if (ret != 0) { + ret = -op_errno; + } + goto out; } else { sys_ret = sys_lsetxattr(real_path, key, value->data, value->len, flags); #ifdef GF_DARWIN_HOST_OS @@ -1822,8 +1843,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp) return 0; } - ret = posix_handle_pair(filler->this, filler->real_path, k, v, XATTR_CREATE, - filler->stbuf); + ret = posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v, + XATTR_CREATE, filler->stbuf); if (ret < 0) { errno = -ret; return -1; @@ -1832,7 +1853,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp) } int -posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict) +posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path, + dict_t *dict) { int ret = -1; @@ -1846,6 +1868,7 @@ posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict) filler.this = this; filler.real_path = path; filler.stbuf = NULL; + filler.loc = loc; ret = dict_foreach(dict, _handle_entry_create_keyvalue_pair, &filler); diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c index 497502e5d42..396bd883383 100644 --- a/xlators/storage/posix/src/posix-inode-fd-ops.c +++ b/xlators/storage/posix/src/posix-inode-fd-ops.c @@ -425,22 +425,9 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, &frame->root->ctime, stbuf, valid); } - if (valid & GF_SET_ATTR_CTIME && !priv->ctime) { - /* - * If ctime is not enabled, we have no means to associate an - * arbitrary ctime with the file, so as a fallback, we ignore - * the ctime payload and update the file ctime to current time - * (which is possible directly with the POSIX API). - */ - op_ret = PATH_SET_TIMESPEC_OR_TIMEVAL(real_path, NULL); - if (op_ret == -1) { - op_errno = errno; - gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_UTIMES_FAILED, - "setattr (utimes) on gfid-handle %s (path: %s) " - "failed", - real_path, loc->path); - goto out; - } + if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) { + posix_update_ctime_in_mdata(this, real_path, -1, loc->inode, + &frame->root->ctime, stbuf, valid); } if (!valid) { @@ -466,14 +453,6 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, goto out; } - if (valid & GF_SET_ATTR_CTIME && priv->ctime) { - /* - * If we got ctime payload, we override - * the ctime of statpost with that. - */ - statpost.ia_ctime = stbuf->ia_ctime; - statpost.ia_ctime_nsec = stbuf->ia_ctime_nsec; - } posix_set_ctime(frame, this, real_path, -1, loc->inode, &statpost); if (xdata) @@ -589,6 +568,7 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt statpost = { 0, }; + struct posix_private *priv = NULL; struct posix_fd *pfd = NULL; dict_t *xattr_rsp = NULL; int32_t ret = -1; @@ -601,6 +581,9 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, VALIDATE_OR_GOTO(this, out); VALIDATE_OR_GOTO(fd, out); + priv = this->private; + VALIDATE_OR_GOTO(priv, out); + ret = posix_fd_ctx_get(fd, this, &pfd, &op_errno); if (ret < 0) { gf_msg_debug(this->name, 0, "pfd is NULL from fd=%p", fd); @@ -653,6 +636,11 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, &frame->root->ctime, stbuf, valid); } + if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) { + posix_update_ctime_in_mdata(this, NULL, pfd->fd, fd->inode, + &frame->root->ctime, stbuf, valid); + } + if (!valid) { op_ret = sys_fchown(pfd->fd, -1, -1); if (op_ret == -1) { @@ -2584,7 +2572,7 @@ _handle_setxattr_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp) filler = tmp; - return posix_handle_pair(filler->this, filler->real_path, k, v, + return posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v, filler->flags, filler->stbuf); } @@ -2647,27 +2635,27 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, priv = this->private; DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, op_ret, op_errno, out); + MAKE_INODE_HANDLE(real_path, this, loc, NULL); + if (!real_path) { + op_ret = -1; + op_errno = ESTALE; + goto out; + } + ret = dict_get_mdata(dict, CTIME_MDATA_XDATA_KEY, &mdata_iatt); if (ret == 0) { /* This is initiated by lookup when ctime feature is enabled to create * "trusted.glusterfs.mdata" xattr if not present. These are the files * which were created when ctime feature is disabled. */ - ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, &mdata_iatt, - &op_errno); + ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path, + &mdata_iatt, &op_errno); if (ret != 0) { op_ret = -1; } goto out; } - MAKE_INODE_HANDLE(real_path, this, loc, NULL); - if (!real_path) { - op_ret = -1; - op_errno = ESTALE; - goto out; - } - posix_pstat(this, loc->inode, loc->gfid, real_path, &preop, _gf_false); op_ret = -1; @@ -2802,6 +2790,7 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, filler.real_path = real_path; filler.this = this; filler.stbuf = &preop; + filler.loc = loc; #ifdef GF_DARWIN_HOST_OS filler.flags = map_xattr_flags(flags); diff --git a/xlators/storage/posix/src/posix-metadata.c b/xlators/storage/posix/src/posix-metadata.c index 532daa26936..9efaf999402 100644 --- a/xlators/storage/posix/src/posix-metadata.c +++ b/xlators/storage/posix/src/posix-metadata.c @@ -56,6 +56,19 @@ posix_mdata_from_disk(posix_mdata_t *out, posix_mdata_disk_t *in) out->atime.tv_nsec = be64toh(in->atime.tv_nsec); } +void +posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in) +{ + out->ia_ctime = be64toh(in->ctime.tv_sec); + out->ia_ctime_nsec = be64toh(in->ctime.tv_nsec); + + out->ia_mtime = be64toh(in->mtime.tv_sec); + out->ia_mtime_nsec = be64toh(in->mtime.tv_nsec); + + out->ia_atime = be64toh(in->atime.tv_sec); + out->ia_atime_nsec = be64toh(in->atime.tv_nsec); +} + /* posix_fetch_mdata_xattr fetches the posix_mdata_t from disk */ static int posix_fetch_mdata_xattr(xlator_t *this, const char *real_path_arg, int _fd, @@ -341,6 +354,7 @@ posix_compare_timespec(struct timespec *first, struct timespec *second) int posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode, + const char *realpath, struct mdata_iatt *mdata_iatt, int *op_errno) { posix_mdata_t *mdata = NULL; @@ -369,8 +383,8 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode, goto unlock; } - ret = posix_fetch_mdata_xattr(this, NULL, -1, inode, (void *)mdata, - op_errno); + ret = posix_fetch_mdata_xattr(this, realpath, -1, inode, + (void *)mdata, op_errno); if (ret == 0) { /* Got mdata from disk. This is a race, another client * has healed the xattr during lookup. So set it in inode @@ -412,7 +426,7 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode, } } - ret = posix_store_mdata_xattr(this, NULL, -1, inode, mdata); + ret = posix_store_mdata_xattr(this, realpath, -1, inode, mdata); if (ret) { gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STOREMDATA_FAILED, "gfid: %s key:%s ", uuid_utoa(inode->gfid), @@ -445,7 +459,8 @@ posix_set_mdata_xattr(xlator_t *this, const char *real_path, int fd, GF_VALIDATE_OR_GOTO(this->name, inode, out); GF_VALIDATE_OR_GOTO(this->name, time, out); - if (update_utime && (!u_atime || !u_mtime)) { + if (update_utime && (flag->ctime && !time) && (flag->atime && !u_atime) && + (flag->mtime && !u_mtime)) { goto out; } @@ -652,6 +667,48 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd, return; } +/* posix_update_ctime_in_mdata updates the posix_mdata_t when ctime needs + * to be modified + */ +void +posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd, + inode_t *inode, struct timespec *ctime, + struct iatt *stbuf, int valid) +{ + int32_t ret = 0; +#if defined(HAVE_UTIMENSAT) + struct timespec tv_ctime = { + 0, + }; +#else + struct timeval tv_ctime = { + 0, + }; +#endif + posix_mdata_flag_t flag = { + 0, + }; + + struct posix_private *priv = NULL; + priv = this->private; + + if (inode && priv->ctime) { + tv_ctime.tv_sec = stbuf->ia_ctime; + SET_TIMESPEC_NSEC_OR_TIMEVAL_USEC(tv_ctime, stbuf->ia_ctime_nsec); + flag.ctime = 1; + + ret = posix_set_mdata_xattr(this, real_path, -1, inode, &tv_ctime, NULL, + NULL, NULL, &flag, _gf_true); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_SETMDATA_FAILED, + "posix set mdata atime failed on file:" + " %s gfid:%s", + real_path, uuid_utoa(inode->gfid)); + } + } + return; +} + static void posix_get_mdata_flag(uint64_t flags, posix_mdata_flag_t *flag) { diff --git a/xlators/storage/posix/src/posix-metadata.h b/xlators/storage/posix/src/posix-metadata.h index c17669974d2..63e8771d3b1 100644 --- a/xlators/storage/posix/src/posix-metadata.h +++ b/xlators/storage/posix/src/posix-metadata.h @@ -43,6 +43,10 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd, inode_t *inode, struct timespec *ctime, struct iatt *stbuf, int valid); void +posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd, + inode_t *inode, struct timespec *ctime, + struct iatt *stbuf, int valid); +void posix_set_ctime(call_frame_t *frame, xlator_t *this, const char *real_path, int fd, inode_t *inode, struct iatt *stbuf); void @@ -56,7 +60,10 @@ posix_set_ctime_cfr(call_frame_t *frame, xlator_t *this, int fd_out, inode_t *inode_out, struct iatt *stbuf_out); int posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode, + const char *realpath, struct mdata_iatt *mdata_iatt, int *op_errno); +void +posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in); #endif /* _POSIX_METADATA_H */ diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h index 9d670625d94..7bacad162cf 100644 --- a/xlators/storage/posix/src/posix.h +++ b/xlators/storage/posix/src/posix.h @@ -336,7 +336,7 @@ dict_t * posix_xattr_fill(xlator_t *this, const char *path, loc_t *loc, fd_t *fd, int fdnum, dict_t *xattr, struct iatt *buf); int -posix_handle_pair(xlator_t *this, const char *real_path, char *key, +posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key, data_t *value, int flags, struct iatt *stbuf); int posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key, @@ -349,7 +349,8 @@ int posix_gfid_heal(xlator_t *this, const char *path, loc_t *loc, dict_t *xattr_req); int -posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict); +posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path, + dict_t *dict); int posix_fd_ctx_get(fd_t *fd, xlator_t *this, struct posix_fd **pfd, -- cgit