ctime/rebalance: Heal ctime xattr on directory during rebalance

After add-brick and rebalance, the ctime xattr is not present
on rebalanced directories on new brick. This patch fixes the
same.

Note that ctime still doesn't support consistent time across
distribute sub-volume.

This patch also fixes the in-memory inconsistency of time attributes
when metadata is self healed.

Backport of:
 > Patch: https://review.gluster.org/23127
 > Change-Id: Ia20506f1839021bf61d4753191e7dc34b31bb2df
 > BUG: 1734026
 > Signed-off-by: Kotresh HR <khiremat@redhat.com>

Patch: https://review.gluster.org/23127
Change-Id: Ia20506f1839021bf61d4753191e7dc34b31bb2df
fixes: bz#1752413
Signed-off-by: Kotresh HR <khiremat@redhat.com>

9 files changed, 132 insertions, 52 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 5115248aefa..2a00f5f9bfa 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -505,7 +505,8 @@ afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
 
     AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
                &replies[source].poststat,
-               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME), NULL);
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
 
     loc_wipe(&loc);
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index ad06d89cc30..f9e7141604a 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -115,6 +115,7 @@ char *xattrs_to_heal[] = {"user.",
                           QUOTA_LIMIT_KEY,
                           QUOTA_LIMIT_OBJECTS_KEY,
                           GF_SELINUX_XATTR_KEY,
+                          GF_XATTR_MDATA_KEY,
                           NULL};
 
 char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index 2fa1f11c5fa..9b716b8c209 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -2301,9 +2301,10 @@ ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
 
         loc.inode = inode_ref(fd->inode);
         gf_uuid_copy(loc.gfid, fd->inode->gfid);
-        ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, &source_buf,
-                              GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL);
+        ret = cluster_setattr(
+            ec->xl_list, healed_sinks, ec->nodes, replies, output, frame,
+            ec->xl, &loc, &source_buf,
+            GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL);
         EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
         if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
             ret = -ENOTCONN;
diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c
index e98647d76bd..55676777373 100644
--- a/xlators/storage/posix/src/posix-entry-ops.c
+++ b/xlators/storage/posix/src/posix-entry-ops.c
@@ -499,7 +499,7 @@ post_op:
         posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name);
     }
 
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         if (errno != EEXIST)
             gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
@@ -827,7 +827,7 @@ posix_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                "setting ACLs on %s failed ", real_path);
     }
 
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed", real_path);
@@ -1528,7 +1528,7 @@ posix_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
     }
 
 ignore:
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed ", real_path);
@@ -2166,7 +2166,7 @@ posix_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name);
     }
 ignore:
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed ", real_path);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index 00690b4a141..e8713b41002 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1189,11 +1189,15 @@ posix_dump_buffer(xlator_t *this, const char *real_path, const char *key,
 #endif
 
 int
-posix_handle_pair(xlator_t *this, const char *real_path, char *key,
+posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key,
                   data_t *value, int flags, struct iatt *stbuf)
 {
     int sys_ret = -1;
     int ret = 0;
+    int op_errno = 0;
+    struct mdata_iatt mdata_iatt = {
+        0,
+    };
 #ifdef GF_DARWIN_HOST_OS
     const int error_code = EINVAL;
 #else
@@ -1217,6 +1221,23 @@ posix_handle_pair(xlator_t *this, const char *real_path, char *key,
         /* ignore this key value pair */
         ret = 0;
         goto out;
+    } else if (!strncmp(key, GF_XATTR_MDATA_KEY, strlen(key))) {
+        /* This is either by rebalance or self heal. Create the xattr if it's
+         * not present. Compare and update the larger value if the xattr is
+         * already present.
+         */
+        if (loc == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        posix_mdata_iatt_from_disk(&mdata_iatt,
+                                   (posix_mdata_disk_t *)value->data);
+        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path,
+                                                 &mdata_iatt, &op_errno);
+        if (ret != 0) {
+            ret = -op_errno;
+        }
+        goto out;
     } else {
         sys_ret = sys_lsetxattr(real_path, key, value->data, value->len, flags);
 #ifdef GF_DARWIN_HOST_OS
@@ -1795,8 +1816,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
         return 0;
     }
 
-    ret = posix_handle_pair(filler->this, filler->real_path, k, v, XATTR_CREATE,
-                            filler->stbuf);
+    ret = posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v,
+                            XATTR_CREATE, filler->stbuf);
     if (ret < 0) {
         errno = -ret;
         return -1;
@@ -1805,7 +1826,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
 }
 
 int
-posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict)
+posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path,
+                             dict_t *dict)
 {
     int ret = -1;
 
@@ -1819,6 +1841,7 @@ posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict)
     filler.this = this;
     filler.real_path = path;
     filler.stbuf = NULL;
+    filler.loc = loc;
 
     ret = dict_foreach(dict, _handle_entry_create_keyvalue_pair, &filler);
 
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index b2e9626b9f1..076f0050cf4 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -369,22 +369,9 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
                                     &frame->root->ctime, stbuf, valid);
     }
 
-    if (valid & GF_SET_ATTR_CTIME && !priv->ctime) {
-        /*
-         * If ctime is not enabled, we have no means to associate an
-         * arbitrary ctime with the file, so as a fallback, we ignore
-         * the ctime payload and update the file ctime to current time
-         * (which is possible directly with the POSIX API).
-         */
-        op_ret = PATH_SET_TIMESPEC_OR_TIMEVAL(real_path, NULL);
-        if (op_ret == -1) {
-            op_errno = errno;
-            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_UTIMES_FAILED,
-                   "setattr (utimes) on %s "
-                   "failed",
-                   real_path);
-            goto out;
-        }
+    if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) {
+        posix_update_ctime_in_mdata(this, real_path, -1, loc->inode,
+                                    &frame->root->ctime, stbuf, valid);
     }
 
     if (!valid) {
@@ -409,14 +396,6 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
         goto out;
     }
 
-    if (valid & GF_SET_ATTR_CTIME && priv->ctime) {
-        /*
-         * If we got ctime payload, we override
-         * the ctime of statpost with that.
-         */
-        statpost.ia_ctime = stbuf->ia_ctime;
-        statpost.ia_ctime_nsec = stbuf->ia_ctime_nsec;
-    }
     posix_set_ctime(frame, this, real_path, -1, loc->inode, &statpost);
 
     if (xdata)
@@ -530,6 +509,7 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
     struct iatt statpost = {
         0,
     };
+    struct posix_private *priv = NULL;
     struct posix_fd *pfd = NULL;
     dict_t *xattr_rsp = NULL;
     int32_t ret = -1;
@@ -542,6 +522,9 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
 
+    priv = this->private;
+    VALIDATE_OR_GOTO(priv, out);
+
     ret = posix_fd_ctx_get(fd, this, &pfd, &op_errno);
     if (ret < 0) {
         gf_msg_debug(this->name, 0, "pfd is NULL from fd=%p", fd);
@@ -594,6 +577,11 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
                                     &frame->root->ctime, stbuf, valid);
     }
 
+    if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) {
+        posix_update_ctime_in_mdata(this, NULL, pfd->fd, fd->inode,
+                                    &frame->root->ctime, stbuf, valid);
+    }
+
     if (!valid) {
         op_ret = sys_fchown(pfd->fd, -1, -1);
         if (op_ret == -1) {
@@ -2488,7 +2476,7 @@ _handle_setxattr_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
 
     filler = tmp;
 
-    return posix_handle_pair(filler->this, filler->real_path, k, v,
+    return posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v,
                              filler->flags, filler->stbuf);
 }
 
@@ -2553,27 +2541,27 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     priv = this->private;
     DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, op_ret, op_errno, out);
 
+    MAKE_INODE_HANDLE(real_path, this, loc, NULL);
+    if (!real_path) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        goto out;
+    }
+
     ret = dict_get_mdata(dict, CTIME_MDATA_XDATA_KEY, &mdata_iatt);
     if (ret == 0) {
         /* This is initiated by lookup when ctime feature is enabled to create
          * "trusted.glusterfs.mdata" xattr if not present. These are the files
          * which were created when ctime feature is disabled.
          */
-        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, &mdata_iatt,
-                                                 &op_errno);
+        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path,
+                                                 &mdata_iatt, &op_errno);
         if (ret != 0) {
             op_ret = -1;
         }
         goto out;
     }
 
-    MAKE_INODE_HANDLE(real_path, this, loc, NULL);
-    if (!real_path) {
-        op_ret = -1;
-        op_errno = ESTALE;
-        goto out;
-    }
-
     posix_pstat(this, loc->inode, loc->gfid, real_path, &preop, _gf_false);
 
     op_ret = -1;
@@ -2682,6 +2670,7 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     filler.real_path = real_path;
     filler.this = this;
     filler.stbuf = &preop;
+    filler.loc = loc;
 
 #ifdef GF_DARWIN_HOST_OS
     filler.flags = map_xattr_flags(flags);
diff --git a/xlators/storage/posix/src/posix-metadata.c b/xlators/storage/posix/src/posix-metadata.c
index 532daa26936..9efaf999402 100644
--- a/xlators/storage/posix/src/posix-metadata.c
+++ b/xlators/storage/posix/src/posix-metadata.c
@@ -56,6 +56,19 @@ posix_mdata_from_disk(posix_mdata_t *out, posix_mdata_disk_t *in)
     out->atime.tv_nsec = be64toh(in->atime.tv_nsec);
 }
 
+void
+posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in)
+{
+    out->ia_ctime = be64toh(in->ctime.tv_sec);
+    out->ia_ctime_nsec = be64toh(in->ctime.tv_nsec);
+
+    out->ia_mtime = be64toh(in->mtime.tv_sec);
+    out->ia_mtime_nsec = be64toh(in->mtime.tv_nsec);
+
+    out->ia_atime = be64toh(in->atime.tv_sec);
+    out->ia_atime_nsec = be64toh(in->atime.tv_nsec);
+}
+
 /* posix_fetch_mdata_xattr fetches the posix_mdata_t from disk */
 static int
 posix_fetch_mdata_xattr(xlator_t *this, const char *real_path_arg, int _fd,
@@ -341,6 +354,7 @@ posix_compare_timespec(struct timespec *first, struct timespec *second)
 
 int
 posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   const char *realpath,
                                    struct mdata_iatt *mdata_iatt, int *op_errno)
 {
     posix_mdata_t *mdata = NULL;
@@ -369,8 +383,8 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
                 goto unlock;
             }
 
-            ret = posix_fetch_mdata_xattr(this, NULL, -1, inode, (void *)mdata,
-                                          op_errno);
+            ret = posix_fetch_mdata_xattr(this, realpath, -1, inode,
+                                          (void *)mdata, op_errno);
             if (ret == 0) {
                 /* Got mdata from disk. This is a race, another client
                  * has healed the xattr during lookup. So set it in inode
@@ -412,7 +426,7 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
             }
         }
 
-        ret = posix_store_mdata_xattr(this, NULL, -1, inode, mdata);
+        ret = posix_store_mdata_xattr(this, realpath, -1, inode, mdata);
         if (ret) {
             gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STOREMDATA_FAILED,
                    "gfid: %s key:%s ", uuid_utoa(inode->gfid),
@@ -445,7 +459,8 @@ posix_set_mdata_xattr(xlator_t *this, const char *real_path, int fd,
     GF_VALIDATE_OR_GOTO(this->name, inode, out);
     GF_VALIDATE_OR_GOTO(this->name, time, out);
 
-    if (update_utime && (!u_atime || !u_mtime)) {
+    if (update_utime && (flag->ctime && !time) && (flag->atime && !u_atime) &&
+        (flag->mtime && !u_mtime)) {
         goto out;
     }
 
@@ -652,6 +667,48 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd,
     return;
 }
 
+/* posix_update_ctime_in_mdata updates the posix_mdata_t when ctime needs
+ * to be modified
+ */
+void
+posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd,
+                            inode_t *inode, struct timespec *ctime,
+                            struct iatt *stbuf, int valid)
+{
+    int32_t ret = 0;
+#if defined(HAVE_UTIMENSAT)
+    struct timespec tv_ctime = {
+        0,
+    };
+#else
+    struct timeval tv_ctime = {
+        0,
+    };
+#endif
+    posix_mdata_flag_t flag = {
+        0,
+    };
+
+    struct posix_private *priv = NULL;
+    priv = this->private;
+
+    if (inode && priv->ctime) {
+        tv_ctime.tv_sec = stbuf->ia_ctime;
+        SET_TIMESPEC_NSEC_OR_TIMEVAL_USEC(tv_ctime, stbuf->ia_ctime_nsec);
+        flag.ctime = 1;
+
+        ret = posix_set_mdata_xattr(this, real_path, -1, inode, &tv_ctime, NULL,
+                                    NULL, NULL, &flag, _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_SETMDATA_FAILED,
+                   "posix set mdata atime failed on file:"
+                   " %s gfid:%s",
+                   real_path, uuid_utoa(inode->gfid));
+        }
+    }
+    return;
+}
+
 static void
 posix_get_mdata_flag(uint64_t flags, posix_mdata_flag_t *flag)
 {
diff --git a/xlators/storage/posix/src/posix-metadata.h b/xlators/storage/posix/src/posix-metadata.h
index c17669974d2..63e8771d3b1 100644
--- a/xlators/storage/posix/src/posix-metadata.h
+++ b/xlators/storage/posix/src/posix-metadata.h
@@ -43,6 +43,10 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd,
                             inode_t *inode, struct timespec *ctime,
                             struct iatt *stbuf, int valid);
 void
+posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd,
+                            inode_t *inode, struct timespec *ctime,
+                            struct iatt *stbuf, int valid);
+void
 posix_set_ctime(call_frame_t *frame, xlator_t *this, const char *real_path,
                 int fd, inode_t *inode, struct iatt *stbuf);
 void
@@ -56,7 +60,10 @@ posix_set_ctime_cfr(call_frame_t *frame, xlator_t *this,
                     int fd_out, inode_t *inode_out, struct iatt *stbuf_out);
 int
 posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   const char *realpath,
                                    struct mdata_iatt *mdata_iatt,
                                    int *op_errno);
+void
+posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in);
 
 #endif /* _POSIX_METADATA_H */
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index d5ba08ca593..395437b8dfa 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -335,7 +335,7 @@ dict_t *
 posix_xattr_fill(xlator_t *this, const char *path, loc_t *loc, fd_t *fd,
                  int fdnum, dict_t *xattr, struct iatt *buf);
 int
-posix_handle_pair(xlator_t *this, const char *real_path, char *key,
+posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key,
                   data_t *value, int flags, struct iatt *stbuf);
 int
 posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key,
@@ -348,7 +348,8 @@ int
 posix_gfid_heal(xlator_t *this, const char *path, loc_t *loc,
                 dict_t *xattr_req);
 int
-posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict);
+posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path,
+                             dict_t *dict);
 
 int
 posix_fd_ctx_get(fd_t *fd, xlator_t *this, struct posix_fd **pfd,