ctime/rebalance: Heal ctime xattr on directory during rebalance

After add-brick and rebalance, the ctime xattr is not present
on rebalanced directories on new brick. This patch fixes the
same.

Note that ctime still doesn't support consistent time across
distribute sub-volume.

This patch also fixes the in-memory inconsistency of time attributes
when metadata is self healed.

Backport of:

 > Patch: https://review.gluster.org/23127/
 > Change-Id: Ia20506f1839021bf61d4753191e7dc34b31bb2df
 > BUG: 1734026
 > Signed-off-by: Kotresh HR <khiremat@redhat.com>
(cherry picked from commit 304640e55c0f3c6d15f4e230dc6376e4f5020fea)

Change-Id: Ia20506f1839021bf61d4753191e7dc34b31bb2df
Signed-off-by: Kotresh HR <khiremat@redhat.com>
fixes: bz#1752429

9 files changed, 132 insertions, 52 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 9e42a83debc..ae03c9c913f 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -513,7 +513,8 @@ afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
 
     AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
                &replies[source].poststat,
-               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME), NULL);
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
 
     loc_wipe(&loc);
 
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index 61e0318473a..d9c6af27298 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -113,6 +113,7 @@ char *xattrs_to_heal[] = {"user.",
                           QUOTA_LIMIT_KEY,
                           QUOTA_LIMIT_OBJECTS_KEY,
                           GF_SELINUX_XATTR_KEY,
+                          GF_XATTR_MDATA_KEY,
                           NULL};
 
 char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index 9f66189e9e0..a868230eb40 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -2309,9 +2309,10 @@ ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
 
         loc.inode = inode_ref(fd->inode);
         gf_uuid_copy(loc.gfid, fd->inode->gfid);
-        ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, replies,
-                              output, frame, ec->xl, &loc, &source_buf,
-                              GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL);
+        ret = cluster_setattr(
+            ec->xl_list, healed_sinks, ec->nodes, replies, output, frame,
+            ec->xl, &loc, &source_buf,
+            GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL);
         EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
         if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
             ret = -ENOTCONN;
diff --git a/xlators/storage/posix/src/posix-entry-ops.c b/xlators/storage/posix/src/posix-entry-ops.c
index 3c070de3929..30b80f6e4a0 100644
--- a/xlators/storage/posix/src/posix-entry-ops.c
+++ b/xlators/storage/posix/src/posix-entry-ops.c
@@ -494,7 +494,7 @@ post_op:
         posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name);
     }
 
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         if (errno != EEXIST)
             gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
@@ -822,7 +822,7 @@ posix_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                "setting ACLs on %s failed ", real_path);
     }
 
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed", real_path);
@@ -1523,7 +1523,7 @@ posix_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
     }
 
 ignore:
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed ", real_path);
@@ -2161,7 +2161,7 @@ posix_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         posix_set_gfid2path_xattr(this, real_path, loc->pargfid, loc->name);
     }
 ignore:
-    op_ret = posix_entry_create_xattr_set(this, real_path, xdata);
+    op_ret = posix_entry_create_xattr_set(this, loc, real_path, xdata);
     if (op_ret) {
         gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_XATTR_FAILED,
                "setting xattrs on %s failed ", real_path);
diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index abbd5917060..d2beeed1f4a 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1184,11 +1184,15 @@ posix_dump_buffer(xlator_t *this, const char *real_path, const char *key,
 #endif
 
 int
-posix_handle_pair(xlator_t *this, const char *real_path, char *key,
+posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key,
                   data_t *value, int flags, struct iatt *stbuf)
 {
     int sys_ret = -1;
     int ret = 0;
+    int op_errno = 0;
+    struct mdata_iatt mdata_iatt = {
+        0,
+    };
 #ifdef GF_DARWIN_HOST_OS
     const int error_code = EINVAL;
 #else
@@ -1212,6 +1216,23 @@ posix_handle_pair(xlator_t *this, const char *real_path, char *key,
         /* ignore this key value pair */
         ret = 0;
         goto out;
+    } else if (!strncmp(key, GF_XATTR_MDATA_KEY, strlen(key))) {
+        /* This is either by rebalance or self heal. Create the xattr if it's
+         * not present. Compare and update the larger value if the xattr is
+         * already present.
+         */
+        if (loc == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        posix_mdata_iatt_from_disk(&mdata_iatt,
+                                   (posix_mdata_disk_t *)value->data);
+        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path,
+                                                 &mdata_iatt, &op_errno);
+        if (ret != 0) {
+            ret = -op_errno;
+        }
+        goto out;
     } else {
         sys_ret = sys_lsetxattr(real_path, key, value->data, value->len, flags);
 #ifdef GF_DARWIN_HOST_OS
@@ -1806,8 +1827,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
         return 0;
     }
 
-    ret = posix_handle_pair(filler->this, filler->real_path, k, v, XATTR_CREATE,
-                            filler->stbuf);
+    ret = posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v,
+                            XATTR_CREATE, filler->stbuf);
     if (ret < 0) {
         errno = -ret;
         return -1;
@@ -1816,7 +1837,8 @@ _handle_entry_create_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
 }
 
 int
-posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict)
+posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path,
+                             dict_t *dict)
 {
     int ret = -1;
 
@@ -1830,6 +1852,7 @@ posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict)
     filler.this = this;
     filler.real_path = path;
     filler.stbuf = NULL;
+    filler.loc = loc;
 
     ret = dict_foreach(dict, _handle_entry_create_keyvalue_pair, &filler);
 
diff --git a/xlators/storage/posix/src/posix-inode-fd-ops.c b/xlators/storage/posix/src/posix-inode-fd-ops.c
index 15e3dc5f33e..d27311ebe6a 100644
--- a/xlators/storage/posix/src/posix-inode-fd-ops.c
+++ b/xlators/storage/posix/src/posix-inode-fd-ops.c
@@ -427,22 +427,9 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
                                     &frame->root->ctime, stbuf, valid);
     }
 
-    if (valid & GF_SET_ATTR_CTIME && !priv->ctime) {
-        /*
-         * If ctime is not enabled, we have no means to associate an
-         * arbitrary ctime with the file, so as a fallback, we ignore
-         * the ctime payload and update the file ctime to current time
-         * (which is possible directly with the POSIX API).
-         */
-        op_ret = PATH_SET_TIMESPEC_OR_TIMEVAL(real_path, NULL);
-        if (op_ret == -1) {
-            op_errno = errno;
-            gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_UTIMES_FAILED,
-                   "setattr (utimes) on gfid-handle %s (path: %s) "
-                   "failed",
-                   real_path, loc->path);
-            goto out;
-        }
+    if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) {
+        posix_update_ctime_in_mdata(this, real_path, -1, loc->inode,
+                                    &frame->root->ctime, stbuf, valid);
     }
 
     if (!valid) {
@@ -468,14 +455,6 @@ posix_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
         goto out;
     }
 
-    if (valid & GF_SET_ATTR_CTIME && priv->ctime) {
-        /*
-         * If we got ctime payload, we override
-         * the ctime of statpost with that.
-         */
-        statpost.ia_ctime = stbuf->ia_ctime;
-        statpost.ia_ctime_nsec = stbuf->ia_ctime_nsec;
-    }
     posix_set_ctime(frame, this, real_path, -1, loc->inode, &statpost);
 
     if (xdata)
@@ -591,6 +570,7 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
     struct iatt statpost = {
         0,
     };
+    struct posix_private *priv = NULL;
     struct posix_fd *pfd = NULL;
     dict_t *xattr_rsp = NULL;
     int32_t ret = -1;
@@ -603,6 +583,9 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
     VALIDATE_OR_GOTO(this, out);
     VALIDATE_OR_GOTO(fd, out);
 
+    priv = this->private;
+    VALIDATE_OR_GOTO(priv, out);
+
     ret = posix_fd_ctx_get(fd, this, &pfd, &op_errno);
     if (ret < 0) {
         gf_msg_debug(this->name, 0, "pfd is NULL from fd=%p", fd);
@@ -655,6 +638,11 @@ posix_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
                                     &frame->root->ctime, stbuf, valid);
     }
 
+    if ((valid & GF_SET_ATTR_CTIME) && priv->ctime) {
+        posix_update_ctime_in_mdata(this, NULL, pfd->fd, fd->inode,
+                                    &frame->root->ctime, stbuf, valid);
+    }
+
     if (!valid) {
         op_ret = sys_fchown(pfd->fd, -1, -1);
         if (op_ret == -1) {
@@ -2582,7 +2570,7 @@ _handle_setxattr_keyvalue_pair(dict_t *d, char *k, data_t *v, void *tmp)
 
     filler = tmp;
 
-    return posix_handle_pair(filler->this, filler->real_path, k, v,
+    return posix_handle_pair(filler->this, filler->loc, filler->real_path, k, v,
                              filler->flags, filler->stbuf);
 }
 
@@ -2645,27 +2633,27 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     priv = this->private;
     DISK_SPACE_CHECK_AND_GOTO(frame, priv, xdata, op_ret, op_errno, out);
 
+    MAKE_INODE_HANDLE(real_path, this, loc, NULL);
+    if (!real_path) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        goto out;
+    }
+
     ret = dict_get_mdata(dict, CTIME_MDATA_XDATA_KEY, &mdata_iatt);
     if (ret == 0) {
         /* This is initiated by lookup when ctime feature is enabled to create
          * "trusted.glusterfs.mdata" xattr if not present. These are the files
          * which were created when ctime feature is disabled.
          */
-        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, &mdata_iatt,
-                                                 &op_errno);
+        ret = posix_set_mdata_xattr_legacy_files(this, loc->inode, real_path,
+                                                 &mdata_iatt, &op_errno);
         if (ret != 0) {
             op_ret = -1;
         }
         goto out;
     }
 
-    MAKE_INODE_HANDLE(real_path, this, loc, NULL);
-    if (!real_path) {
-        op_ret = -1;
-        op_errno = ESTALE;
-        goto out;
-    }
-
     posix_pstat(this, loc->inode, loc->gfid, real_path, &preop, _gf_false);
 
     op_ret = -1;
@@ -2800,6 +2788,7 @@ posix_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
     filler.real_path = real_path;
     filler.this = this;
     filler.stbuf = &preop;
+    filler.loc = loc;
 
 #ifdef GF_DARWIN_HOST_OS
     filler.flags = map_xattr_flags(flags);
diff --git a/xlators/storage/posix/src/posix-metadata.c b/xlators/storage/posix/src/posix-metadata.c
index 532daa26936..9efaf999402 100644
--- a/xlators/storage/posix/src/posix-metadata.c
+++ b/xlators/storage/posix/src/posix-metadata.c
@@ -56,6 +56,19 @@ posix_mdata_from_disk(posix_mdata_t *out, posix_mdata_disk_t *in)
     out->atime.tv_nsec = be64toh(in->atime.tv_nsec);
 }
 
+void
+posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in)
+{
+    out->ia_ctime = be64toh(in->ctime.tv_sec);
+    out->ia_ctime_nsec = be64toh(in->ctime.tv_nsec);
+
+    out->ia_mtime = be64toh(in->mtime.tv_sec);
+    out->ia_mtime_nsec = be64toh(in->mtime.tv_nsec);
+
+    out->ia_atime = be64toh(in->atime.tv_sec);
+    out->ia_atime_nsec = be64toh(in->atime.tv_nsec);
+}
+
 /* posix_fetch_mdata_xattr fetches the posix_mdata_t from disk */
 static int
 posix_fetch_mdata_xattr(xlator_t *this, const char *real_path_arg, int _fd,
@@ -341,6 +354,7 @@ posix_compare_timespec(struct timespec *first, struct timespec *second)
 
 int
 posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   const char *realpath,
                                    struct mdata_iatt *mdata_iatt, int *op_errno)
 {
     posix_mdata_t *mdata = NULL;
@@ -369,8 +383,8 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
                 goto unlock;
             }
 
-            ret = posix_fetch_mdata_xattr(this, NULL, -1, inode, (void *)mdata,
-                                          op_errno);
+            ret = posix_fetch_mdata_xattr(this, realpath, -1, inode,
+                                          (void *)mdata, op_errno);
             if (ret == 0) {
                 /* Got mdata from disk. This is a race, another client
                  * has healed the xattr during lookup. So set it in inode
@@ -412,7 +426,7 @@ posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
             }
         }
 
-        ret = posix_store_mdata_xattr(this, NULL, -1, inode, mdata);
+        ret = posix_store_mdata_xattr(this, realpath, -1, inode, mdata);
         if (ret) {
             gf_msg(this->name, GF_LOG_ERROR, errno, P_MSG_STOREMDATA_FAILED,
                    "gfid: %s key:%s ", uuid_utoa(inode->gfid),
@@ -445,7 +459,8 @@ posix_set_mdata_xattr(xlator_t *this, const char *real_path, int fd,
     GF_VALIDATE_OR_GOTO(this->name, inode, out);
     GF_VALIDATE_OR_GOTO(this->name, time, out);
 
-    if (update_utime && (!u_atime || !u_mtime)) {
+    if (update_utime && (flag->ctime && !time) && (flag->atime && !u_atime) &&
+        (flag->mtime && !u_mtime)) {
         goto out;
     }
 
@@ -652,6 +667,48 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd,
     return;
 }
 
+/* posix_update_ctime_in_mdata updates the posix_mdata_t when ctime needs
+ * to be modified
+ */
+void
+posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd,
+                            inode_t *inode, struct timespec *ctime,
+                            struct iatt *stbuf, int valid)
+{
+    int32_t ret = 0;
+#if defined(HAVE_UTIMENSAT)
+    struct timespec tv_ctime = {
+        0,
+    };
+#else
+    struct timeval tv_ctime = {
+        0,
+    };
+#endif
+    posix_mdata_flag_t flag = {
+        0,
+    };
+
+    struct posix_private *priv = NULL;
+    priv = this->private;
+
+    if (inode && priv->ctime) {
+        tv_ctime.tv_sec = stbuf->ia_ctime;
+        SET_TIMESPEC_NSEC_OR_TIMEVAL_USEC(tv_ctime, stbuf->ia_ctime_nsec);
+        flag.ctime = 1;
+
+        ret = posix_set_mdata_xattr(this, real_path, -1, inode, &tv_ctime, NULL,
+                                    NULL, NULL, &flag, _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, errno, P_MSG_SETMDATA_FAILED,
+                   "posix set mdata atime failed on file:"
+                   " %s gfid:%s",
+                   real_path, uuid_utoa(inode->gfid));
+        }
+    }
+    return;
+}
+
 static void
 posix_get_mdata_flag(uint64_t flags, posix_mdata_flag_t *flag)
 {
diff --git a/xlators/storage/posix/src/posix-metadata.h b/xlators/storage/posix/src/posix-metadata.h
index c17669974d2..63e8771d3b1 100644
--- a/xlators/storage/posix/src/posix-metadata.h
+++ b/xlators/storage/posix/src/posix-metadata.h
@@ -43,6 +43,10 @@ posix_update_utime_in_mdata(xlator_t *this, const char *real_path, int fd,
                             inode_t *inode, struct timespec *ctime,
                             struct iatt *stbuf, int valid);
 void
+posix_update_ctime_in_mdata(xlator_t *this, const char *real_path, int fd,
+                            inode_t *inode, struct timespec *ctime,
+                            struct iatt *stbuf, int valid);
+void
 posix_set_ctime(call_frame_t *frame, xlator_t *this, const char *real_path,
                 int fd, inode_t *inode, struct iatt *stbuf);
 void
@@ -56,7 +60,10 @@ posix_set_ctime_cfr(call_frame_t *frame, xlator_t *this,
                     int fd_out, inode_t *inode_out, struct iatt *stbuf_out);
 int
 posix_set_mdata_xattr_legacy_files(xlator_t *this, inode_t *inode,
+                                   const char *realpath,
                                    struct mdata_iatt *mdata_iatt,
                                    int *op_errno);
+void
+posix_mdata_iatt_from_disk(struct mdata_iatt *out, posix_mdata_disk_t *in);
 
 #endif /* _POSIX_METADATA_H */
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index da590b53e1b..806d431afab 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -333,7 +333,7 @@ dict_t *
 posix_xattr_fill(xlator_t *this, const char *path, loc_t *loc, fd_t *fd,
                  int fdnum, dict_t *xattr, struct iatt *buf);
 int
-posix_handle_pair(xlator_t *this, const char *real_path, char *key,
+posix_handle_pair(xlator_t *this, loc_t *loc, const char *real_path, char *key,
                   data_t *value, int flags, struct iatt *stbuf);
 int
 posix_fhandle_pair(call_frame_t *frame, xlator_t *this, int fd, char *key,
@@ -346,7 +346,8 @@ int
 posix_gfid_heal(xlator_t *this, const char *path, loc_t *loc,
                 dict_t *xattr_req);
 int
-posix_entry_create_xattr_set(xlator_t *this, const char *path, dict_t *dict);
+posix_entry_create_xattr_set(xlator_t *this, loc_t *loc, const char *path,
+                             dict_t *dict);
 
 int
 posix_fd_ctx_get(fd_t *fd, xlator_t *this, struct posix_fd **pfd,