diff options
Diffstat (limited to 'tests/functional/dht/test_add_brick_rebalance_with_rsync_in_progress.py')
| -rw-r--r-- | tests/functional/dht/test_add_brick_rebalance_with_rsync_in_progress.py | 151 |
1 files changed, 151 insertions, 0 deletions
diff --git a/tests/functional/dht/test_add_brick_rebalance_with_rsync_in_progress.py b/tests/functional/dht/test_add_brick_rebalance_with_rsync_in_progress.py new file mode 100644 index 000000000..799ce1a60 --- /dev/null +++ b/tests/functional/dht/test_add_brick_rebalance_with_rsync_in_progress.py @@ -0,0 +1,151 @@ +# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along` +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from glusto.core import Glusto as g +from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on +from glustolibs.gluster.exceptions import ExecutionError +from glustolibs.gluster.glusterdir import mkdir +from glustolibs.gluster.rebalance_ops import ( + rebalance_start, wait_for_rebalance_to_complete) +from glustolibs.gluster.volume_libs import expand_volume +from glustolibs.io.utils import collect_mounts_arequal, run_linux_untar + + +@runs_on([['distributed-replicated', 'distributed-arbiter', + 'distributed-dispersed', 'distributed'], ['glusterfs']]) +class TestAddBrickRebalanceWithRsyncInProgress(GlusterBaseClass): + + def setUp(self): + # calling GlusterBaseClass setUp + self.get_super_method(self, 'setUp')() + + # Changing dist_count to 3 + self.volume['voltype']['dist_count'] = 3 + + # Set I/O flag to false + self.is_io_running = False + + # Creating Volume and mounting the volume + ret = self.setup_volume_and_mount_volume([self.mounts[0]]) + if not ret: + raise ExecutionError("Volume creation or mount failed: %s" + % self.volname) + + def tearDown(self): + + # Wait for I/O if not completed + if self.is_io_running: + if not self._wait_for_untar_and_rsync_completion(): + g.log.error("I/O failed to stop on clients") + + # Unmounting and cleaning volume + ret = self.unmount_volume_and_cleanup_volume([self.mounts[0]]) + if not ret: + raise ExecutionError("Unable to delete volume % s" % self.volname) + + self.get_super_method(self, 'tearDown')() + + def _wait_for_untar_and_rsync_completion(self): + """Wait for untar and rsync to complete""" + has_process_stopped = [] + for proc in self.list_of_io_processes: + try: + ret, _, _ = proc.async_communicate() + if not ret: + has_process_stopped.append(False) + has_process_stopped.append(True) + except ValueError: + has_process_stopped.append(True) + return all(has_process_stopped) + + def test_add_brick_rebalance_with_rsync_in_progress(self): + """ + Test case: + 1. Create, start and mount a volume. + 2. Create a directory on the mount point and start linux utar. + 3. Create another directory on the mount point and start rsync of + linux untar directory. + 4. Add bricks to the volume + 5. Trigger rebalance on the volume. + 6. Wait for rebalance to complete on volume. + 7. Wait for I/O to complete. + 8. Validate if checksum of both the untar and rsync is same. + """ + # List of I/O processes + self.list_of_io_processes = [] + + # Create a dir to start untar + self.linux_untar_dir = "{}/{}".format(self.mounts[0].mountpoint, + "linuxuntar") + ret = mkdir(self.clients[0], self.linux_untar_dir) + self.assertTrue(ret, "Failed to create dir linuxuntar for untar") + + # Start linux untar on dir linuxuntar + ret = run_linux_untar(self.clients[0], self.mounts[0].mountpoint, + dirs=tuple(['linuxuntar'])) + self.list_of_io_processes += ret + self.is_io_running = True + + # Create a new directory and start rsync + self.rsync_dir = "{}/{}".format(self.mounts[0].mountpoint, + 'rsyncuntarlinux') + ret = mkdir(self.clients[0], self.rsync_dir) + self.assertTrue(ret, "Failed to create dir rsyncuntarlinux for rsync") + + # Start rsync for linux untar on mount point + cmd = ("for i in `seq 1 3`; do rsync -azr {} {};sleep 120;done" + .format(self.linux_untar_dir, self.rsync_dir)) + ret = g.run_async(self.clients[0], cmd) + self.list_of_io_processes.append(ret) + + # Add bricks to the volume + ret = expand_volume(self.mnode, self.volname, self.servers, + self.all_servers_info) + self.assertTrue(ret, "Failed to add brick with rsync on volume %s" + % self.volname) + + # Trigger rebalance on the volume + ret, _, _ = rebalance_start(self.mnode, self.volname) + self.assertEqual(ret, 0, "Failed to start rebalance on the volume %s" + % self.volname) + + # Wait for rebalance to complete + ret = wait_for_rebalance_to_complete(self.mnode, self.volname, + timeout=6000) + self.assertTrue(ret, "Rebalance is not yet complete on the volume " + "%s" % self.volname) + + # Wait for IO to complete. + ret = self._wait_for_untar_and_rsync_completion() + self.assertFalse(ret, "IO didn't complete or failed on client") + self.is_io_running = False + + # As we are running rsync and untar together, there are situations + # when some of the new files created by linux untar is not synced + # through rsync which causes checksum to retrun different value, + # Hence to take care of this corner case we are rerunning rsync. + cmd = "rsync -azr {} {}".format(self.linux_untar_dir, self.rsync_dir) + ret, _, _ = g.run(self.clients[0], cmd) + self.assertEqual(ret, 0, "Failed sync left behind files") + + # Check daata consistency on both the directories + rsync_checksum = collect_mounts_arequal( + self.mounts[0], path='rsyncuntarlinux/linuxuntar/') + untar_checksum = collect_mounts_arequal(self.mounts[0], + path='linuxuntar') + self.assertEqual( + rsync_checksum, untar_checksum, + "Checksum on untar dir and checksum on rsync dir didn't match") |