summaryrefslogtreecommitdiffstats
path: root/tests/functional/afr
diff options
context:
space:
mode:
authorBala Konda Reddy M <bala12352@gmail.com>2020-08-17 17:27:34 +0530
committerBala Konda Reddy M <bala12352@gmail.com>2020-08-21 08:58:05 +0000
commit5568547036c1953e1cc25e6f32f652380ba525c9 (patch)
tree2dce5c055d24de4401b8a0c9a251c02de7ff4013 /tests/functional/afr
parent4ec7f7791aafb94f4c4b161aad30cd6c9d4f4d5a (diff)
[Test] Test conservative merge between two bricks
Test Steps: 1) Create 1x3 volume and fuse mount the volume 2) On mount created a dir dir1 3) Pkill glusterfsd on node n1 (b2 on node2 and b3 and node3 up) 4) touch f{1..10} on the mountpoint 5) b2 and b3 xattrs would be blaming b1 as files are created while b1 is down 6) Reset the b3 xattrs to NOT blame b1 by using setattr 7) Now pkill glusterfsd of b2 on node2 8) Restart glusterd on node1 to bring up b1 9) Now bricks b1 online , b2 down, b3 online 10) touch x{1..10} under dir1 itself 11) Again reset xattr on node3 of b3 so that it doesn't blame b2, as done for b1 in step 6 12) Do restart glusterd on node2 hosting b2 to bring all bricks online 13) Check for heal info, split-brain and arequal for the bricks Change-Id: Ieea875dd7243c7f8d2c6959aebde220508134d7a Signed-off-by: Bala Konda Reddy M <bala12352@gmail.com>
Diffstat (limited to 'tests/functional/afr')
-rw-r--r--tests/functional/afr/heal/test_heal_for_conservative_merge_with_two_bricks_blame.py175
1 files changed, 175 insertions, 0 deletions
diff --git a/tests/functional/afr/heal/test_heal_for_conservative_merge_with_two_bricks_blame.py b/tests/functional/afr/heal/test_heal_for_conservative_merge_with_two_bricks_blame.py
new file mode 100644
index 0000000..163596b
--- /dev/null
+++ b/tests/functional/afr/heal/test_heal_for_conservative_merge_with_two_bricks_blame.py
@@ -0,0 +1,175 @@
+# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from time import sleep
+
+from glusto.core import Glusto as g
+from glustolibs.gluster.brick_libs import (get_all_bricks, are_bricks_offline,
+ bring_bricks_offline,
+ get_online_bricks_list,
+ are_bricks_online)
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.glusterdir import mkdir
+from glustolibs.gluster.gluster_init import restart_glusterd
+from glustolibs.gluster.glusterfile import set_fattr, get_fattr
+from glustolibs.gluster.heal_libs import (is_volume_in_split_brain,
+ monitor_heal_completion)
+from glustolibs.gluster.lib_utils import collect_bricks_arequal
+
+
+@runs_on([['replicated'], ['glusterfs']])
+class TestHealForConservativeMergeWithTwoBricksBlame(GlusterBaseClass):
+
+ def setUp(self):
+ # calling GlusterBaseClass setUp
+ self.get_super_method(self, 'setUp')()
+
+ # Setup volume and mount it.
+ if not self.setup_volume_and_mount_volume([self.mounts[0]]):
+ raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
+
+ def tearDown(self):
+ # Unmount and cleanup the volume
+ if not self.unmount_volume_and_cleanup_volume([self.mounts[0]]):
+ raise ExecutionError("Unable to unmount and cleanup volume")
+
+ # Calling GlusterBaseClass tearDown
+ self.get_super_method(self, 'tearDown')()
+
+ def _bring_brick_offline_and_check(self, brick):
+ """Brings brick offline an checks if it is offline or not"""
+ ret = bring_bricks_offline(self.volname, [brick])
+ self.assertTrue(ret, "Unable to bring brick: {} offline".format(brick))
+
+ # Validate the brick is offline
+ ret = are_bricks_offline(self.mnode, self.volname, [brick])
+ self.assertTrue(ret, "Brick:{} is still online".format(brick))
+
+ def _get_fattr_for_the_brick(self, brick):
+ """Get xattr of trusted.afr.volname-client-0 for the given brick"""
+ host, fqpath = brick.split(":")
+ fqpath = fqpath + "/dir1"
+ fattr = "trusted.afr.{}-client-0".format(self.volname)
+ return get_fattr(host, fqpath, fattr, encode="hex")
+
+ def _check_peers_status(self):
+ """Validates peers are connected or not"""
+ count = 0
+ while count < 4:
+ if self.validate_peers_are_connected():
+ return
+ sleep(5)
+ count += 1
+ self.fail("Peers are not in connected state")
+
+ def test_heal_for_conservative_merge_with_two_bricks_blame(self):
+ """
+ 1) Create 1x3 volume and fuse mount the volume
+ 2) On mount created a dir dir1
+ 3) Pkill glusterfsd on node n1 (b2 on node2 and b3 and node3 up)
+ 4) touch f{1..10} on the mountpoint
+ 5) b2 and b3 xattrs would be blaming b1 as files are created while
+ b1 is down
+ 6) Reset the b3 xattrs to NOT blame b1 by using setattr
+ 7) Now pkill glusterfsd of b2 on node2
+ 8) Restart glusterd on node1 to bring up b1
+ 9) Now bricks b1 online , b2 down, b3 online
+ 10) touch x{1..10} under dir1 itself
+ 11) Again reset xattr on node3 of b3 so that it doesn't blame b2,
+ as done for b1 in step 6
+ 12) Do restart glusterd on node2 hosting b2 to bring all bricks online
+ 13) Check for heal info, split-brain and arequal for the bricks
+ """
+ # pylint: disable=too-many-locals
+ # Create dir `dir1/` on mountpont
+ path = self.mounts[0].mountpoint + "/dir1"
+ ret = mkdir(self.mounts[0].client_system, path, parents=True)
+ self.assertTrue(ret, "Directory {} creation failed".format(path))
+
+ all_bricks = get_all_bricks(self.mnode, self.volname)
+ self.assertIsNotNone(all_bricks, "Unable to fetch bricks of volume")
+ brick1, brick2, brick3 = all_bricks
+
+ # Bring first brick offline
+ self._bring_brick_offline_and_check(brick1)
+
+ # touch f{1..10} files on the mountpoint
+ cmd = ("cd {mpt}; for i in `seq 1 10`; do touch f$i"
+ "; done".format(mpt=path))
+ ret, _, _ = g.run(self.mounts[0].client_system, cmd)
+ self.assertEqual(ret, 0, "Unable to create files on mountpoint")
+
+ # Check b2 and b3 xattrs are blaming b1 and are same
+ self.assertEqual(self._get_fattr_for_the_brick(brick2),
+ self._get_fattr_for_the_brick(brick3),
+ "Both the bricks xattrs are not blaming "
+ "brick: {}".format(brick1))
+
+ # Reset the xattrs of dir1 on b3 for brick b1
+ first_xattr_to_reset = "trusted.afr.{}-client-0".format(self.volname)
+ xattr_value = "0x000000000000000000000000"
+ host, brick_path = brick3.split(":")
+ brick_path = brick_path + "/dir1"
+ ret = set_fattr(host, brick_path, first_xattr_to_reset, xattr_value)
+ self.assertTrue(ret, "Unable to set xattr for the directory")
+
+ # Kill brick2 on the node2
+ self._bring_brick_offline_and_check(brick2)
+
+ # Restart glusterd on node1 to bring the brick1 online
+ self.assertTrue(restart_glusterd([brick1.split(":")[0]]), "Unable to "
+ "restart glusterd")
+ # checking for peer status post glusterd restart
+ self._check_peers_status()
+
+ # Check if the brick b1 on node1 is online or not
+ online_bricks = get_online_bricks_list(self.mnode, self.volname)
+ self.assertIsNotNone(online_bricks, "Unable to fetch online bricks")
+ self.assertIn(brick1, online_bricks, "Brick:{} is still offline after "
+ "glusterd restart".format(brick1))
+
+ # Create 10 files under dir1 naming x{1..10}
+ cmd = ("cd {mpt}; for i in `seq 1 10`; do touch x$i"
+ "; done".format(mpt=path))
+ ret, _, _ = g.run(self.mounts[0].client_system, cmd)
+ self.assertEqual(ret, 0, "Unable to create files on mountpoint")
+
+ # Reset the xattrs from brick3 on to brick2
+ second_xattr_to_reset = "trusted.afr.{}-client-1".format(self.volname)
+ ret = set_fattr(host, brick_path, second_xattr_to_reset, xattr_value)
+ self.assertTrue(ret, "Unable to set xattr for the directory")
+
+ # Bring brick2 online
+ self.assertTrue(restart_glusterd([brick2.split(":")[0]]), "Unable to "
+ "restart glusterd")
+ self._check_peers_status()
+
+ self.assertTrue(are_bricks_online(self.mnode, self.volname, [brick2]))
+
+ # Check are there any files in split-brain and heal completion
+ self.assertFalse(is_volume_in_split_brain(self.mnode, self.volname),
+ "Some files are in split brain for "
+ "volume: {}".format(self.volname))
+ self.assertTrue(monitor_heal_completion(self.mnode, self.volname),
+ "Conservative merge of files failed")
+
+ # Check arequal checksum of all the bricks is same
+ ret, arequal_from_the_bricks = collect_bricks_arequal(all_bricks)
+ self.assertTrue(ret, "Arequal is collected successfully across the"
+ " bricks in the subvol {}".format(all_bricks))
+ self.assertEqual(len(set(arequal_from_the_bricks)), 1, "Arequal is "
+ "same on all the bricks in the subvol")