afr: remove stale index entries

Backport of http://review.gluster.org/9714

Problem:
During pre-op phase, the index xlator
1. Creates the entry inside .glusterfs/indices/xattrop
2. Winds the xattrop fop to posix to mark dirty/pending changelogs.
If the brick crashes after 1, the xattrop entry becomes stale and never
gets removed by shd during subsequent crawls because there is nothing to
heal (changelogs are zero).

Though the stale entry does not get displayed in the output of 'heal info'
command, it nevertheless stays there forever unless a new write tansaction
is performed on the file.

Fix:
During index self-heal if afr xattrs are found to be clean (indicated by
ret value of 2 on a call to afr_shd_selfheal(), send a dummy
post-op with all 0s for the xattr values, which makes the index xlator
to unlink the stale entry.

Change-Id: Iffb171e40490abd8d44df09ccc058b5da67baafe
BUG: 1203081
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/9920
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Reviewed-by: Raghavendra Bhat <raghavendra@redhat.com>

2 files changed, 66 insertions, 3 deletions

diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index c6b14e1def7..7fb6e2b9dc4 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1141,6 +1141,11 @@ afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid)
 	if (ret)
 		goto out;
 
+        if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+                ret = 2;
+                goto out;
+        }
+
 	if (data_selfheal)
                 data_ret = afr_selfheal_data (frame, this, inode);
 
@@ -1169,9 +1174,12 @@ out:
         return ret;
 }
 /*
- * This is the entry point for healing a given GFID
- * The function returns 0 if self-heal was successful, appropriate errno
- * in case of a failure and 1 in case self-heal was never needed on the gfid.
+ * This is the entry point for healing a given GFID. The return values for this
+ * function are as follows:
+ * '0' if the self-heal is successful
+ * '1' if the afr-xattrs are non-zero (due to on-going IO) and no heal is needed
+ * '2' if the afr-xattrs are all-zero and no heal is needed
+ * $errno if the heal on the gfid failed.
  */
 
 int
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 992ed28b994..7c235aca429 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -313,6 +313,53 @@ afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
 	return ret;
 }
 
+void
+afr_shd_zero_xattrop (xlator_t *this, uuid_t gfid)
+{
+
+        call_frame_t *frame = NULL;
+        inode_t *inode = NULL;
+        afr_private_t *priv = NULL;
+        dict_t  *xattr = NULL;
+        int ret = 0;
+        int i = 0;
+        int raw[AFR_NUM_CHANGE_LOGS] = {0};
+
+        priv = this->private;
+        frame = afr_frame_create (this);
+        if (!frame)
+                goto out;
+        inode = afr_inode_find (this, gfid);
+        if (!inode)
+                goto out;
+        xattr = dict_new();
+        if (!xattr)
+                goto out;
+        ret = dict_set_static_bin (xattr, AFR_DIRTY, raw,
+                                   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret)
+                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_set_static_bin (xattr, priv->pending_key[i], raw,
+                                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+                if (ret)
+                        goto out;
+        }
+
+        /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
+        * has valid repies for this gfid seems a bit of an overkill.*/
+        for (i = 0; i < priv->child_count; i++)
+                afr_selfheal_post_op (frame, this, inode, i, xattr);
+
+out:
+        if (frame)
+                AFR_STACK_DESTROY (frame);
+        if (inode)
+                inode_unref (inode);
+        if (xattr)
+                dict_unref (xattr);
+        return;
+}
 
 int
 afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
@@ -480,6 +527,14 @@ afr_shd_index_sweep (struct subvol_healer *healer)
 				afr_shd_index_purge (subvol, fd->inode,
 						     entry->d_name);
 			}
+                        if (ret == 2)
+                        /* If bricks crashed in pre-op after creating
+                         * indices/xattrop link but before setting afr
+                         * changelogs, we end up with stale xattrop links but
+                         * zero changelogs. Remove such entries by sending a
+                         * post-op with zero changelogs.
+                         */
+                                afr_shd_zero_xattrop (healer->this, gfid);
                         ret = 0;
 		}