[Test] Validate data, metadata, entry split-brain

Steps:
- Create and mount a replicated volume and disable quorum,
self-heal deamon
- Create ~10 files from the mount point and simulate data, metadata
split-brain for 2 files each
- Create a dir with some files and simulate entry/gfid split brain
- Validate volume successfully recognizing split-brain
- Validate a lookup on split-brain files fails with EIO error on mount
- Validate `heal info` and `heal info split-brain` command shows only
the files that are in split-brain
- Validate new files and dir's can be created from the mount

Change-Id: I8caeb284c53304a74473815ae5181213c710b085
Signed-off-by: Leela Venkaiah G <lgangava@redhat.com>

1 files changed, 264 insertions, 0 deletions

diff --git a/tests/functional/afr/test_heal_split_brain_command.py b/tests/functional/afr/test_heal_split_brain_command.py
new file mode 100644
index 000000000..c924e8910
--- /dev/null
+++ b/tests/functional/afr/test_heal_split_brain_command.py
@@ -0,0 +1,264 @@
+#  Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from glusto.core import Glusto as g
+
+from glustolibs.gluster.brick_libs import (bring_bricks_offline,
+                                           bring_bricks_online, get_all_bricks)
+from glustolibs.gluster.exceptions import ExecutionError
+from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
+from glustolibs.gluster.glusterdir import mkdir
+from glustolibs.gluster.glusterfile import get_fattr
+from glustolibs.gluster.heal_libs import is_volume_in_split_brain
+from glustolibs.gluster.heal_ops import heal_info, heal_info_split_brain
+from glustolibs.gluster.volume_ops import set_volume_options
+
+
+# pylint: disable=too-many-locals, too-many-statements
+@runs_on([['arbiter', 'replicated'], ['glusterfs']])
+class TestSplitBrain(GlusterBaseClass):
+    def setUp(self):
+        self.get_super_method(self, 'setUp')()
+
+        # A single mount is enough for all the tests
+        self.mounts = [self.mounts[0]]
+
+        if not self.setup_volume_and_mount_volume(mounts=self.mounts):
+            raise ExecutionError('Failed to setup and mount '
+                                 '{}'.format(self.volname))
+
+    def tearDown(self):
+        if not self.unmount_volume_and_cleanup_volume(mounts=self.mounts):
+            raise ExecutionError('Not able to unmount and cleanup '
+                                 '{}'.format(self.volname))
+        self.get_super_method(self, 'tearDown')()
+
+    def _run_cmd_and_validate(self, client, cmd, paths):
+        """
+        Run `cmd` from `paths` on `client`
+        """
+        for path in paths:
+            ret, _, _ = g.run(client, cmd % path)
+            self.assertEqual(
+                ret, 0, 'Unable to perform `{}` from `{}` on `{}`'.format(
+                    cmd, path, client))
+
+    @staticmethod
+    def _transform_gfids(gfids):
+        """
+        Returns  list of `gfids` joined by `-` at required places
+
+        Example of one elemnt:
+        Input:   0xd4653ea0289548eb81b35c91ffb73eff
+        Returns: d4653ea0-2895-48eb-81b3-5c91ffb73eff
+        """
+        split_pos = [10, 14, 18, 22]
+        rout = []
+        for gfid in gfids:
+            rout.append('-'.join(
+                gfid[start:stop]
+                for start, stop in zip([2] + split_pos, split_pos + [None])))
+        return rout
+
+    def test_split_brain_from_heal_command(self):
+        """
+        Description: Simulate and validate data, metadata and entry split brain
+
+        Steps:
+        - Create and mount a replicated volume and disable quorum, self-heal
+          deamon
+        - Create ~10 files from the mount point and simulate data, metadata
+          split-brain for 2 files each
+        - Create a dir with some files and simulate entry/gfid split brain
+        - Validate volume successfully recognizing split-brain
+        - Validate a lookup on split-brain files fails with EIO error on mount
+        - Validate `heal info` and `heal info split-brain` command shows only
+          the files that are in split-brain
+        - Validate new files and dir's can be created from the mount
+        """
+        io_cmd = 'cat /dev/urandom | tr -dc [:space:][:print:] | head -c '
+        client, m_point = (self.mounts[0].client_system,
+                           self.mounts[0].mountpoint)
+        arbiter = self.volume_type.find('arbiter') >= 0
+
+        # Disable self-heal daemon and set `quorum-type` option to `none`
+        ret = set_volume_options(self.mnode, self.volname, {
+            'self-heal-daemon': 'off',
+            'cluster.quorum-type': 'none'
+        })
+        self.assertTrue(
+            ret, 'Not able to disable `quorum-type` and '
+            '`self-heal` daemon volume options')
+
+        # Create required dir's from the mount
+        fqpath = '{}/dir'.format(m_point)
+        file_io = ('cd %s; for i in {1..6}; do ' + io_cmd +
+                   ' 2M > file$i; done;')
+        file_cmd = 'cd %s; touch file{7..10}'
+        ret = mkdir(client, fqpath)
+        self.assertTrue(ret, 'Unable to create a directory from mount point')
+
+        # Create empty files and data files
+        for cmd in (file_io, file_cmd):
+            self._run_cmd_and_validate(client, cmd, [m_point, fqpath])
+
+        all_bricks = get_all_bricks(self.mnode, self.volname)
+        self.assertIsNotNone(
+            all_bricks, 'Unable to get list of bricks '
+            'associated with the volume')
+
+        # Data will be appended to the files `file1, file2` resulting in data
+        # split brain
+        data_split_cmd = ';'.join(io_cmd + '2M >> ' + each_file
+                                  for each_file in ('file1', 'file2'))
+
+        # File permissions will be changed for `file4, file5` to result in
+        # metadata split brain
+        meta_split_cmd = ';'.join('chmod 0555 ' + each_file
+                                  for each_file in ('file4', 'file5'))
+
+        # Files will be deleted and created with data to result in data,
+        # metadata split brain on files and entry(gfid) split brain on dir
+        entry_split_cmd = ';'.join('rm -f ' + each_file + ' && ' + io_cmd +
+                                   ' 2M > ' + each_file
+                                   for each_file in ('dir/file1', 'dir/file2'))
+
+        # Need to always select arbiter(3rd) brick if volume is arbiter type or
+        # any two bricks for replicated volume
+        for bricks in zip(all_bricks, all_bricks[1:] + [all_bricks[0]]):
+
+            # Skip iteration if volume type is arbiter and `bricks` doesn't
+            # contain arbiter brick
+            if arbiter and (all_bricks[-1] not in bricks):
+                continue
+
+            # Bring bricks offline
+            ret = bring_bricks_offline(self.volname, list(bricks))
+            self.assertTrue(ret, 'Unable to bring {} offline'.format(bricks))
+
+            # Run cmd to bring files into split brain
+            for cmd, msg in ((data_split_cmd, 'data'),
+                             (meta_split_cmd, 'meta'), (entry_split_cmd,
+                                                        'entry')):
+                ret, _, _ = g.run(client, 'cd {}; {}'.format(m_point, cmd))
+                self.assertEqual(
+                    ret, 0, 'Unable to run cmd for bringing files '
+                    'into {} split brain'.format(msg))
+
+            # Bring offline bricks online
+            ret = bring_bricks_online(
+                self.mnode,
+                self.volname,
+                bricks,
+                bring_bricks_online_methods='volume_start_force')
+            self.assertTrue(ret, 'Unable to bring {} online'.format(bricks))
+
+        # Validate volume is in split-brain
+        self.assertTrue(is_volume_in_split_brain(self.mnode, self.volname),
+                        'Volume should be in split-brain')
+
+        # Validate `head` lookup on split brain files fails with EIO
+        for each_file in ('file1', 'file2', 'file4', 'file5', 'dir/file1',
+                          'dir/file2'):
+            ret, _, err = g.run(client,
+                                'cd {}; head {}'.format(m_point, each_file))
+            self.assertNotEqual(
+                ret, 0, 'Lookup on split-brain file {} should '
+                'fail'.format(each_file))
+            self.assertIn(
+                'Input/output error', err,
+                'File {} should result in EIO error'.format(each_file))
+
+        # Validate presence of split-brain files and absence of other files in
+        # `heal info` and `heal info split-brain` commands
+        ret, info, _ = heal_info(self.mnode, self.volname)
+        self.assertEqual(ret, 0, 'Unable to query for `heal info`')
+        ret, info_spb, _ = heal_info_split_brain(self.mnode, self.volname)
+        self.assertEqual(ret, 0, 'Unable to query for `heal info split-brain`')
+
+        # Collect `gfid's` of files in data and metadata split-brain
+        common_gfids = []
+        host, path = all_bricks[0].split(':')
+        for each_file in ('file1', 'file2', 'file4', 'file5', 'dir'):
+            fattr = get_fattr(host, path + '/{}'.format(each_file),
+                              'trusted.gfid')
+            self.assertIsNotNone(
+                fattr, 'Unable to get `gfid` for {}'.format(each_file))
+            common_gfids.append(fattr)
+
+        # GFID for files under an entry split brain dir differs from it's peers
+        uniq_gfids = []
+        for brick in all_bricks[:-1] if arbiter else all_bricks:
+            host, path = brick.split(':')
+            for each_file in ('dir/file1', 'dir/file2'):
+                fattr = get_fattr(host, path + '/{}'.format(each_file),
+                                  'trusted.gfid')
+                self.assertIsNotNone(
+                    fattr, 'Unable to get `gfid` for {}'.format(each_file))
+                uniq_gfids.append(fattr)
+
+        # Transform GFIDs to match against o/p of `heal info` and `split-brain`
+        common_gfids[:] = self._transform_gfids(common_gfids)
+        uniq_gfids[:] = self._transform_gfids(uniq_gfids)
+
+        # Just enough validation by counting occurences asserting success
+        common_files = ['/file1 -', '/file2 -', '/file4', '/file5', '/dir ']
+        uniq_files = ['/dir/file1', '/dir/file2']
+
+        # Common files should occur 3 times each in `heal info` and
+        # `heal info split-brain` or 2 times for arbiter
+        occur = 2 if arbiter else 3
+        for each_file, gfid in zip(common_files, common_gfids):
+
+            # Check against `heal info` cmd
+            self.assertEqual(
+                info.count(gfid) + info.count(each_file), occur,
+                'File {} with gfid {} should exist in `heal info` '
+                'command'.format(each_file[:6], gfid))
+
+            # Check against `heal info split-brain` cmd
+            self.assertEqual(
+                info_spb.count(gfid) + info_spb.count(each_file[:6].rstrip()),
+                occur, 'File {} with gfid {} should exist in `heal info '
+                'split-brain` command'.format(each_file[:6], gfid))
+
+        # Entry split files will be listed only in `heal info` cmd
+        for index, each_file in enumerate(uniq_files):
+
+            # Collect file and it's associated gfid's
+            entries = (uniq_files + uniq_gfids)[index::2]
+            count = sum(info.count(entry) for entry in entries)
+            self.assertEqual(
+                count, occur, 'Not able to find existence of '
+                'entry split brain file {} in `heal info`'.format(each_file))
+
+        # Assert no other file is counted as in split-brain
+        for cmd, rout, exp_str in (('heal info', info, 'entries: 7'),
+                                   ('heal info split-brain', info_spb,
+                                    'split-brain: 5')):
+            self.assertEqual(
+                rout.count(exp_str), occur, 'Each node should '
+                'list only {} entries in {} command'.format(exp_str[-1], cmd))
+
+        # Validate new files and dir can be created from mount
+        fqpath = '{}/temp'.format(m_point)
+        ret = mkdir(client, fqpath)
+        self.assertTrue(
+            ret, 'Unable to create a dir from mount post split-brain of files')
+        for cmd in (file_io, file_cmd):
+            self._run_cmd_and_validate(client, cmd, [fqpath])
+
+        g.log.info('Pass: Validated data, metadata and entry split brain')