[testfix] Update test_snap_list_after_restart

There can be a case where due to a failed tearDown of a previous
test case, the snapshot creation might fail. This can happen when
a redundant snapshot with the same name is present.

This fix includes the changed snapshot names along with tearDown
for deleting the snapshots created in the test case
"test_snap_list_after_restart.py".

This fix also includes the method 'wait_for_glusterd_to_start'
where there is a restart of glusterd.
Adding a check for validation of snapshots using snapname.

Change-Id: If8b48a12bd067ad54dba742eeb88444beaf5f153
Signed-off-by: Vinayak Papnoi <vpapnoi@redhat.com>

1 files changed, 109 insertions, 88 deletions

diff --git a/tests/functional/snapshot/test_snap_list_after_restart.py b/tests/functional/snapshot/test_snap_list_after_restart.py
index 4b73a2b01..cbac4b04a 100644
--- a/tests/functional/snapshot/test_snap_list_after_restart.py
+++ b/tests/functional/snapshot/test_snap_list_after_restart.py
@@ -1,4 +1,4 @@
-#  Copyright (C) 2017-2018  Red Hat, Inc. <http://www.redhat.com>
+#  Copyright (C) 2017-2020  Red Hat, Inc. <http://www.redhat.com>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -14,116 +14,137 @@
 #  with this program; if not, write to the Free Software Foundation, Inc.,
 #  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-"""
-Description:
-
-Test Cases in this module tests the
-snapshot listing before and after
-glusterd restart.
-
-"""
-import time
 from glusto.core import Glusto as g
 from glustolibs.gluster.exceptions import ExecutionError
 from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
-from glustolibs.gluster.gluster_init import (restart_glusterd,
-                                             is_glusterd_running)
-from glustolibs.gluster.snap_ops import (snap_create, snap_delete,
-                                         get_snap_list)
+from glustolibs.gluster.gluster_init import (
+    wait_for_glusterd_to_start,
+    restart_glusterd)
+from glustolibs.gluster.peer_ops import is_peer_connected
+from glustolibs.gluster.snap_ops import (
+    snap_create,
+    snap_delete,
+    snap_delete_all,
+    get_snap_list)
 
 
 @runs_on([['replicated', 'distributed-replicated', 'dispersed',
            'distributed', 'distributed-dispersed'],
           ['glusterfs']])
-class SnapshotGlusterddown(GlusterBaseClass):
-
+class TestSnapshotListGlusterdRestart(GlusterBaseClass):
+    """
+    Test Cases in this module tests the snapshot listing
+    before and after glusterd restart.
+    """
     def setUp(self):
         # SettingUp volume and Mounting the volume
         self.get_super_method(self, 'setUp')()
-        g.log.info("Starting to SetUp Volume")
-        ret = self.setup_volume_and_mount_volume(mounts=self.mounts)
+        ret = self.setup_volume()
         if not ret:
             raise ExecutionError("Failed to setup volume %s" % self.volname)
         g.log.info("Volume %s has been setup successfully", self.volname)
+        self.snapshots = [('snap-test-snap-list-gd-restart-%s-%s'
+                           % (self.volname, i))for i in range(0, 3)]
 
     def tearDown(self):
 
-        # Unmount and cleanup original volume
-        g.log.info("Starting to Unmount Volume and Cleanup Volume")
-        ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
-        if not ret:
-            raise ExecutionError("Failed to umount the vol & cleanup Volume")
-        g.log.info("Successful in umounting the volume and Cleanup")
+        self.get_super_method(self, 'tearDown')()
 
-    def test_snap_delete_and_list_glusterd_down(self):
-        # pylint: disable=too-many-statements
+        # Delete snapshots created in the test case
+        ret, _, _ = snap_delete_all(self.mnode)
+        if ret:
+            raise ExecutionError("Failed to delete the snapshots")
+        g.log.info("Successfully deleted all snapshots")
 
+        # Unmount and cleanup volume
+        ret = self.cleanup_volume()
+        if not ret:
+            raise ExecutionError("Failed to cleanup Volume")
+        g.log.info("Successful in Cleanup volume")
+
+    def test_snap_list_glusterd_restart(self):
         """
-        Steps:
-
-        1. create a volume
-        2. mount volume
-        3. create 3 snapshot of that volume
-        4. delete snapshot snap1
-        5. list all snapshots created
-        6. restart glusterd
-        7. list all snapshots created
-           except snap1
+        Verify snapshot list before and after glusterd restart
+
+        * Create 3 snapshots of the volume
+        * Delete one snapshot
+        * List all snapshots created
+        * Restart glusterd on all nodes
+        * List all snapshots
+          All snapshots must be listed except the one that was deleted
         """
 
-        # Creating snapshot:
-        g.log.info("Starting to Create snapshot")
-        for snap_count in range(0, 3):
-            self.snap = "snap%s" % snap_count
-            ret, _, _ = snap_create(self.mnode, self.volname, self.snap)
-            self.assertEqual(ret, 0, ("Failed to create snapshot for "
-                                      "volume %s" % self.volname))
+        # pylint: disable=too-many-statements
+        # Create snapshots
+        for snap in self.snapshots:
+            ret, _, _ = snap_create(self.mnode, self.volname, snap)
+            self.assertEqual(ret, 0, ("Failed to create snapshot %s for "
+                                      "volume %s" % (snap, self.volname)))
             g.log.info("Snapshot %s created successfully "
-                       "for volume %s", self.snap, self.volname)
-
-        # delete snap1 snapshot
-        g.log.info("Starting to Delete snapshot snap1")
-        ret, _, _ = snap_delete(self.mnode, "snap1")
-        self.assertEqual(ret, 0, "Failed to delete"
-                         "snapshot snap1")
-        g.log.info("Snapshots snap1 deleted Successfully")
-
-        # snapshot list
-        g.log.info("Starting to list all snapshots")
-        out = get_snap_list(self.mnode)
-        self.assertIsNotNone(out, "Failed to list all snapshots")
-        self.assertEqual(len(out), 2, "Failed to validate snap list")
+                       "for volume %s", snap, self.volname)
+
+        # List the snapshots and validate with snapname
+        snap_list = get_snap_list(self.mnode)
+        self.assertIsNotNone(snap_list, "Failed to list all snapshots")
+        self.assertEqual(len(snap_list), 3, "Failed to validate snap list")
+        g.log.info("Successfully validated snap list")
+        for snap in self.snapshots:
+            self.assertIn(snap, snap_list, "Failed to validate the snapshot "
+                          "%s in the snapshot list" % snap)
+        g.log.info("Successfully validated the presence of snapshots using "
+                   "snapname")
+
+        # Delete one snapshot
+        ret, _, _ = snap_delete(self.mnode, self.snapshots[0])
+        self.assertEqual(ret, 0, ("Failed to delete snapshot %s"
+                                  % self.snapshots[0]))
+        g.log.info("Snapshots %s deleted Successfully", self.snapshots[0])
+
+        # List the snapshots and validate with snapname
+        snap_list = get_snap_list(self.mnode)
+        self.assertIsNotNone(snap_list, "Failed to list all snapshots")
+        self.assertEqual(len(snap_list), 2, "Failed to validate snap list")
         g.log.info("Successfully validated snap list")
+        for snap in self.snapshots[1:]:
+            self.assertIn(snap, snap_list, "Failed to validate the snapshot "
+                          "%s in the snapshot list" % snap)
+        g.log.info("Successfully validated the presence of snapshots using "
+                   "snapname")
 
-        # restart Glusterd
-        g.log.info("Restarting Glusterd on all nodes")
+        # Restart glusterd on all the servers
         ret = restart_glusterd(self.servers)
-        self.assertTrue(ret, "Failed to restart glusterd on nodes"
-                        "%s" % self.servers)
-        g.log.info("Successfully restarted glusterd on nodes"
-                   " %s", self.servers)
-
-        # check glusterd running
-        g.log.info("Checking glusterd is running or not")
-        count = 0
-        while count < 80:
-            ret = is_glusterd_running(self.servers)
-            if ret == 0:
-                break
-            time.sleep(2)
-            count += 1
-
-        self.assertEqual(ret, 0, "Failed to validate glusterd "
-                         "running on nodes %s" % self.servers)
-        g.log.info("glusterd is running on "
-                   "nodes %s", self.servers)
-
-        # snapshot list
-        g.log.info("Starting to list all snapshots")
-        for server in self.servers[0:]:
-            out = get_snap_list(server)
-            self.assertIsNotNone(out, "Failed to list snap in node"
-                                 "%s" % server)
-            self.assertEqual(len(out), 2, "Failed to validate snap list"
-                             "on node %s" % server)
-            g.log.info("Successfully validated snap list on node %s", server)
+        self.assertTrue(ret, ("Failed to restart glusterd on nodes %s"
+                              % self.servers))
+        g.log.info("Successfully restarted glusterd on nodes %s", self.servers)
+
+        # Wait for glusterd to be online and validate glusterd running on all
+        # server nodes
+        self.assertTrue(
+            wait_for_glusterd_to_start(self.servers),
+            "Unexpected: glusterd not up on one or more of the nodes")
+        g.log.info("Glusterd is up and running on all nodes")
+
+        # Check if peers are connected
+        self.assertTrue(
+            is_peer_connected(self.mnode, self.servers),
+            "Unexpected: Peers are not in connected state")
+        g.log.info("Successful: All peers are in connected state")
+
+        # List the snapshots after glusterd restart
+        # All snapshots must be listed except the one deleted
+        for server in self.servers:
+            snap_list = get_snap_list(server)
+            self.assertIsNotNone(
+                snap_list, "Failed to get the list of snapshots in node %s"
+                % server)
+            self.assertEqual(
+                len(snap_list), 2,
+                "Unexpected: Number of snapshots not consistent in the node %s"
+                % server)
+            g.log.info("Successfully validated snap list for node %s", server)
+            for snap in self.snapshots[1:]:
+                self.assertIn(
+                    snap, snap_list, "Failed to validate the snapshot "
+                    "%s in the snapshot list" % snap)
+            g.log.info("Successfully validated the presence of snapshots "
+                       "using snapname for node %s", server)