cluster/afr: For entry self heal, mark all source bricks

Problem:
Whenever a new brick is added into a replicate volume, all
source bricks are not marked as source. Only one of them is
marked as source. Here marked as source refers to adding
extended attribute at the backend of a file corresponding to
the newly added brick. As well as source bricks should point
to the newly added brick so that heal can be triggered.

Fix:
All source bricks will now point to newly added bricks and heal
can be triggered based on the extended attributes.

Change-Id: Ia7cf118270fecc429bdecddbcb9201f23fedc7a1
BUG: 1037501
Signed-off-by: Venkatesh Somyajulu <vsomyaju@redhat.com>
Reviewed-on: http://review.gluster.org/6419
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
Reviewed-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

2 files changed, 269 insertions, 13 deletions

diff --git a/tests/bugs/bug-1037501.t b/tests/bugs/bug-1037501.t
new file mode 100755
index 000000000..5470d0563
--- /dev/null
+++ b/tests/bugs/bug-1037501.t
@@ -0,0 +1,220 @@
+#!/bin/bash
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+function write_file()
+{
+	path="$1"; shift
+	echo "$*" > "$path"
+}
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume info;
+
+## Start and create a volume
+mkdir -p ${B0}/${V0}-0
+mkdir -p ${B0}/${V0}-1
+mkdir -p ${B0}/${V0}-2
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}-{0,1,2}
+
+## Verify volume is created
+EXPECT "$V0" volinfo_field $V0 'Volume Name';
+EXPECT 'Created' volinfo_field $V0 'Status';
+
+## Make sure io-cache and write-behind don't interfere.
+TEST $CLI volume set $V0 data-self-heal off;
+
+## Make sure automatic self-heal doesn't perturb our results.
+TEST $CLI volume set $V0 cluster.self-heal-daemon off
+
+TEST $CLI volume set $V0 background-self-heal-count 0
+
+## Start volume and verify
+TEST $CLI volume start $V0;
+EXPECT 'Started' volinfo_field $V0 'Status';
+
+## Mount native
+TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0
+
+TEST `echo "TEST-FILE" > $M0/File`
+TEST `mkdir $M0/Dir`
+TEST `ln  $M0/File $M0/Link`
+TEST `mknod $M0/FIFO p`
+
+TEST $CLI volume add-brick $V0 replica 4 $H0:$B0/$V0-3 force
+TEST $CLI volume add-brick $V0 replica 5 $H0:$B0/$V0-4 force
+TEST $CLI volume add-brick $V0 replica 6 $H0:$B0/$V0-5 force
+
+sleep 10
+
+TEST ls $M0/
+
+
+function compare()
+{
+	var=-1;
+	if [ $1 == $2 ]; then
+		var=0;
+	else
+		var=-1;
+	fi
+
+	echo $var
+}
+
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/File -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Dir -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000001"
+EXPECT 0 compare $var1 $var2
+
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/Link -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000020000000200000000"
+EXPECT 0 compare $var1 $var2
+
+
+
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-0/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-1/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-3"`
+var2="trusted.afr.$V0-client-3=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-4"`
+var2="trusted.afr.$V0-client-4=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+var1=`getfattr -d -m . $B0/$V0-2/FIFO -e hex 2>&1 | grep "client-5"`
+var2="trusted.afr.$V0-client-5=0x000000000000000100000000"
+EXPECT 0 compare $var1 $var2
+
+cleanup;
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 53491a1d7..2989b2adb 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1002,6 +1002,7 @@ afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
         afr_private_t   *priv = NULL;
         afr_local_t     *impunge_local = NULL;
         int              child_index = 0;
+        int              call_count = -1;
 
         priv          = this->private;
         impunge_local = impunge_frame->local;
@@ -1012,16 +1013,26 @@ afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "%s: failed to perform xattrop on %s (%s)",
                         impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
+                        priv->children[child_index]->name, strerror (op_errno));
+
+                        LOCK (&impunge_frame->lock);
+                        {
+                                impunge_local->op_ret = -1;
+                                impunge_local->op_errno = errno;
+                        }
+                        UNLOCK (&impunge_frame->lock);
         }
 
-        afr_sh_entry_impunge_setattr (impunge_frame, this);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
+        call_count = afr_frame_return (impunge_frame);
+
+        if (call_count == 0) {
+                if (impunge_local->op_ret == 0) {
+                        afr_sh_entry_impunge_setattr (impunge_frame, this);
+                } else {
+                        afr_sh_entry_call_impunge_done (impunge_frame, this,
+                                                -1, impunge_local->op_errno);
+                }
+        }
         return 0;
 }
 
@@ -1035,11 +1046,15 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         afr_local_t     *impunge_local    = NULL;
         afr_self_heal_t *impunge_sh       = NULL;
         int32_t         op_errno          = 0;
+        int32_t         call_count        = 0;
+        int32_t         i                 = 0;
+
 
         priv = this->private;
         impunge_local = impunge_frame->local;
         impunge_sh = &impunge_local->self_heal;
         active_src = impunge_sh->active_source;
+        impunge_local->op_ret = 0;
 
         afr_prepare_new_entry_pending_matrix (impunge_local->pending,
                                               afr_is_errno_unset,
@@ -1055,11 +1070,32 @@ afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
         afr_set_pending_dict (priv, xattr, impunge_local->pending, active_src,
                               LOCAL_LAST);
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        for (i = 0; i < priv->child_count; i++) {
+                if ((impunge_sh->child_errno[i] == EEXIST) &&
+                    (impunge_local->child_up[i] == 1))
+
+                        call_count++;
+        }
+
+        impunge_local->call_count  = call_count;
+
+        for (i = 0; i < priv->child_count; i++) {
+
+                if ((impunge_sh->child_errno[i] == EEXIST)
+                    && (impunge_local->child_up[i] == 1)) {
+
+
+                        STACK_WIND_COOKIE (impunge_frame,
+                                           afr_sh_entry_impunge_xattrop_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->xattrop,
+                                           &impunge_local->loc,
+                                           GF_XATTROP_ADD_ARRAY, xattr, NULL);
+                        if (!--call_count)
+                                break;
+                }
+        }
 
         if (xattr)
                 dict_unref (xattr);