[Test] Creates a split brain and verifies IO failure

Steps:
1. Create a volume and mount it
2. Disable heal and cluster-quorum-count
3. Bring down one data and arbiter brick from one
   subvol
4. Write IO and validate it
5. Bring up bricks
6. Bring down another data brick and arbiter brick
   from the same subvol
7. Write IO and validate it
8. Bring up bricks
9. Check if split-brain is created
10. Write IO -> should fail
11. Enable heal and cluster-quorum-count
12. Write IO -> should fail

Change-Id: I229b58c1bcd70dcd87d35dc410e12f51b032b9c4
Signed-off-by: ubansal <ubansal@redhat.com>

1 files changed, 165 insertions, 0 deletions

diff --git a/tests/functional/arbiter/test_split_brain.py b/tests/functional/arbiter/test_split_brain.py
new file mode 100644
index 000000000..e2684be49
--- /dev/null
+++ b/tests/functional/arbiter/test_split_brain.py
@@ -0,0 +1,165 @@
+#  Copyright (C) 2020  Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# pylint: disable=too-many-statements, too-many-locals
+from glusto.core import Glusto as g
+
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.brick_libs import (bring_bricks_offline,
+                                           bring_bricks_online,
+                                           wait_for_bricks_to_be_online)
+from glustolibs.misc.misc_libs import upload_scripts
+from glustolibs.gluster.volume_ops import set_volume_options
+from glustolibs.io.utils import validate_io_procs
+from glustolibs.gluster.heal_libs import is_volume_in_split_brain
+from glustolibs.gluster.volume_libs import get_subvols
+
+
+@runs_on([['arbiter', 'distributed-arbiter'], ['glusterfs']])
+class TestSplitBrain(GlusterBaseClass):
+
+    @classmethod
+    def setUpClass(cls):
+
+        # Calling GlusterBaseClass setUpClass
+        cls.get_super_method(cls, 'setUpClass')()
+
+        # Upload io scripts for running IO on mounts
+        cls.script_upload_path = ("/usr/share/glustolibs/io/scripts/"
+                                  "file_dir_ops.py")
+        ret = upload_scripts(cls.clients, cls.script_upload_path)
+        if not ret:
+            raise ExecutionError("Failed to upload IO scripts "
+                                 "to clients %s" % cls.clients)
+
+        # Setup Volume and Mount Volume
+        ret = cls.setup_volume_and_mount_volume(cls.mounts, True)
+        if not ret:
+            raise ExecutionError("Failed to Setup_Volume and Mount_Volume")
+
+    @classmethod
+    def tearDownClass(cls):
+        """
+        Cleanup Volume
+        """
+        ret = cls.unmount_volume_and_cleanup_volume(cls.mounts)
+        if not ret:
+            raise ExecutionError("Failed to create volume")
+
+        cls.get_super_method(cls, 'tearDownClass')()
+
+    def _bring_bricks_online(self):
+        """
+        Bring bricks online and monitor heal completion
+        """
+        # Bring bricks online
+        ret = bring_bricks_online(
+            self.mnode,
+            self.volname,
+            self.bricks_to_bring_offline,
+            bring_bricks_online_methods=['volume_start_force'])
+        self.assertTrue(ret, 'Failed to bring bricks online')
+
+        # Wait for volume processes to be online
+        ret = wait_for_bricks_to_be_online(self.mnode, self.volname)
+        self.assertTrue(ret, ("Failed to wait for volume {} processes to "
+                              "be online".format(self.volname)))
+
+    def test_split_brain(self):
+
+        """
+        Description: Create split-brain on files and check if IO's fail
+        - Disable self-heal and cluster-quorum-type
+        - Get the bricks from the volume
+        - Write IO and validate IO
+        - Bring 1st set of brick offline(1 Data brick and arbiter brick)
+        - Write IO and validate IO
+        - Bring 2nd set of bricks offline(1 Data brick and arbiter brick)
+        - Write IO and validate IO
+        - Check volume is in split-brain
+        - Write IO and validate IO - should fail
+        - Enable self-heal and cluster-quorum-type
+        - Write IO and validate IO - should fail
+        """
+        # Disable self-heal and cluster-quorum-type
+        options = {"self-heal-daemon": "off",
+                   "cluster.quorum-type": "none"}
+        ret = set_volume_options(self.mnode, self.volname, options)
+        self.assertTrue(ret, ("Unable to set volume option %s for "
+                              "volume %s" % (options, self.volname)))
+
+        # Get the bricks from the volume
+        sub_vols = get_subvols(self.mnode, self.volname)
+        self.bricks_to_bring_offline = list(sub_vols['volume_subvols'][0])
+
+        # Write IO's
+        write_cmd = ("/usr/bin/env python %s create_files -f 1 "
+                     "--base-file-name test_file --fixed-file-size 1k %s" %
+                     (self.script_upload_path,
+                      self.mounts[0].mountpoint))
+        ret, _, _ = g.run(self.mounts[0].client_system, write_cmd)
+
+        # Bring 1st set of brick offline(1 Data brick and arbiter brick)
+        for bricks in ((0, -1), (1, -1)):
+            down_bricks = []
+            for brick in bricks:
+                down_bricks.append(self.bricks_to_bring_offline[brick])
+            ret = bring_bricks_offline(self.volname, down_bricks)
+            self.assertTrue(ret, 'Failed to bring bricks {} offline'.
+                            format(down_bricks))
+            proc = g.run_async(self.mounts[0].client_system, write_cmd)
+
+            # Validate I/O
+            self.assertTrue(
+                validate_io_procs([proc], self.mounts),
+                "IO failed on some of the clients"
+            )
+
+            # Bring bricks online
+            self._bring_bricks_online()
+
+        # Check volume is in split-brain
+        ret = is_volume_in_split_brain(self.mnode, self.volname)
+        self.assertTrue(ret, "unable to create split-brain scenario")
+        g.log.info("Successfully created split brain scenario")
+
+        # Write IO's
+        proc2 = g.run_async(self.mounts[0].client_system, write_cmd)
+
+        # Validate I/O
+        self.assertFalse(
+            validate_io_procs([proc2], self.mounts),
+            "IO passed on split-brain"
+        )
+        g.log.info("Expected - IO's failed due to split-brain")
+
+        # Enable self-heal and cluster-quorum-type
+        options = {"self-heal-daemon": "on",
+                   "cluster.quorum-type": "auto"}
+        ret = set_volume_options(self.mnode, self.volname, options)
+        self.assertTrue(ret, ("Unable to set volume option %s for "
+                              "volume %s" % (options, self.volname)))
+
+        # Write IO's
+        proc3 = g.run_async(self.mounts[0].client_system, write_cmd)
+
+        # Validate I/O
+        self.assertFalse(
+            validate_io_procs([proc3], self.mounts),
+            "IO passed on split-brain"
+        )
+        g.log.info("Expected - IO's failed due to split-brain")