summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--tests/functional/afr/test_afr_reset_brick_heal_full.py157
1 files changed, 157 insertions, 0 deletions
diff --git a/tests/functional/afr/test_afr_reset_brick_heal_full.py b/tests/functional/afr/test_afr_reset_brick_heal_full.py
new file mode 100644
index 000000000..bdc90ee62
--- /dev/null
+++ b/tests/functional/afr/test_afr_reset_brick_heal_full.py
@@ -0,0 +1,157 @@
+# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from random import choice
+
+from glusto.core import Glusto as g
+from glustolibs.gluster.brick_ops import reset_brick
+from glustolibs.gluster.brick_libs import (get_all_bricks, are_bricks_offline)
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.glusterdir import rmdir
+from glustolibs.gluster.glusterfile import remove_file
+from glustolibs.gluster.heal_ops import trigger_heal_full
+from glustolibs.gluster.heal_libs import monitor_heal_completion
+from glustolibs.gluster.lib_utils import collect_bricks_arequal
+from glustolibs.gluster.volume_libs import (
+ get_subvols, wait_for_volume_process_to_be_online)
+from glustolibs.misc.misc_libs import upload_scripts
+from glustolibs.io.utils import (validate_io_procs, wait_for_io_to_complete)
+
+
+@runs_on([['replicated', 'distributed-replicated'],
+ ['glusterfs', 'nfs']])
+class TestAfrResetBrickHeal(GlusterBaseClass):
+
+ @classmethod
+ def setUpClass(cls):
+ # Calling GlusterBaseClass setUpClass
+ cls.get_super_method(cls, 'setUpClass')()
+
+ # Upload IO scripts for running IO on mounts
+ cls.script_upload_path = (
+ "/usr/share/glustolibs/io/scripts/file_dir_ops.py")
+ ret = upload_scripts(cls.clients, cls.script_upload_path)
+ if not ret:
+ raise ExecutionError("Failed to upload IO scripts to clients {}".
+ format(cls.clients))
+
+ def setUp(self):
+ # calling GlusterBaseClass setUp
+ self.get_super_method(self, 'setUp')()
+
+ # Setup volume and mount it.
+ if not self.setup_volume_and_mount_volume(self.mounts):
+ raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
+
+ def tearDown(self):
+ # Wait if any IOs are pending from the test
+ if self.all_mounts_procs:
+ ret = wait_for_io_to_complete(self.all_mounts_procs, self.mounts)
+ if ret:
+ raise ExecutionError(
+ "Wait for IO completion failed on some of the clients")
+
+ # Unmount and cleanup the volume
+ if not self.unmount_volume_and_cleanup_volume(self.mounts):
+ raise ExecutionError("Unable to unmount and cleanup volume")
+
+ # Calling GlusterBaseClass tearDown
+ self.get_super_method(self, 'tearDown')()
+
+ @classmethod
+ def tearDownClass(cls):
+ for each_client in cls.clients:
+ ret = remove_file(each_client, cls.script_upload_path)
+ if not ret:
+ raise ExecutionError("Failed to delete file {}".
+ format(cls.script_upload_path))
+
+ cls.get_super_method(cls, 'tearDownClass')()
+
+ def test_afr_reset_brick_heal_full(self):
+ """
+ 1. Create files/dirs from mount point
+ 2. With IO in progress execute reset-brick start
+ 3. Now format the disk from back-end, using rm -rf <brick path>
+ 4. Execute reset brick commit and check for the brick is online.
+ 5. Issue volume heal using "gluster vol heal <volname> full"
+ 6. Check arequal for all bricks to verify all backend bricks
+ including the resetted brick have same data
+ """
+ self.all_mounts_procs = []
+ for count, mount_obj in enumerate(self.mounts):
+ cmd = ("/usr/bin/env python %s create_deep_dirs_with_files "
+ "--dirname-start-num %d --dir-depth 3 --dir-length 5 "
+ "--max-num-of-dirs 5 --num-of-files 5 %s" % (
+ self.script_upload_path, count,
+ mount_obj.mountpoint))
+ proc = g.run_async(mount_obj.client_system, cmd,
+ user=mount_obj.user)
+ self.all_mounts_procs.append(proc)
+
+ all_bricks = get_all_bricks(self.mnode, self.volname)
+ self.assertIsNotNone(all_bricks, "Unable to fetch bricks of volume")
+ brick_to_reset = choice(all_bricks)
+
+ # Start reset brick
+ ret, _, err = reset_brick(self.mnode, self.volname,
+ src_brick=brick_to_reset, option="start")
+ self.assertEqual(ret, 0, err)
+ g.log.info("Reset brick: %s started", brick_to_reset)
+
+ # Validate the brick is offline
+ ret = are_bricks_offline(self.mnode, self.volname, [brick_to_reset])
+ self.assertTrue(ret, "Brick:{} is still online".format(brick_to_reset))
+
+ # rm -rf of the brick directory
+ node, brick_path = brick_to_reset.split(":")
+ ret = rmdir(node, brick_path, force=True)
+ self.assertTrue(ret, "Unable to delete the brick {} on "
+ "node {}".format(brick_path, node))
+
+ # Reset brick commit
+ ret, _, err = reset_brick(self.mnode, self.volname,
+ src_brick=brick_to_reset, option="commit")
+ self.assertEqual(ret, 0, err)
+ g.log.info("Reset brick committed successfully")
+
+ # Check the brick is online
+ ret = wait_for_volume_process_to_be_online(self.mnode, self.volname)
+ self.assertTrue(ret, "Few volume processess are offline for the "
+ "volume: {}".format(self.volname))
+
+ # Trigger full heal
+ ret = trigger_heal_full(self.mnode, self.volname)
+ self.assertTrue(ret, "Unable to trigger the heal full command")
+
+ # Wait for the heal completion
+ ret = monitor_heal_completion(self.mnode, self.volname)
+ self.assertTrue(ret, "Heal didn't complete in 20 mins time")
+
+ # Validate io on the clients
+ ret = validate_io_procs(self.all_mounts_procs, self.mounts)
+ self.assertTrue(ret, "IO failed on the mounts")
+ self.all_mounts_procs *= 0
+
+ # Check arequal of the back-end bricks after heal completion
+ all_subvols = get_subvols(self.mnode, self.volname)['volume_subvols']
+ for subvol in all_subvols:
+ ret, arequal_from_subvol = collect_bricks_arequal(subvol)
+ self.assertTrue(ret, "Arequal is collected successfully across the"
+ " bricks in the subvol {}".format(subvol))
+ self.assertEqual(len(set(arequal_from_subvol)), 1, "Arequal is "
+ "same on all the bricks in the subvol")