Refactor 'test_disabling_device' test module

The single test in this module passes only in small amount of Heketi
cluster states. So, refactor it completely to be reliable making it fail
only on proper errors.

Change-Id: I29251d4265ec5916d41bd3d69e21492f89097337

2 files changed, 125 insertions, 115 deletions

diff --git a/cns-libs/cnslibs/common/heketi_ops.py b/cns-libs/cnslibs/common/heketi_ops.py
index 8c090586..9b1b29e3 100644
--- a/cns-libs/cnslibs/common/heketi_ops.py
+++ b/cns-libs/cnslibs/common/heketi_ops.py
@@ -443,8 +443,9 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
     Returns:
         dict: volume create info on success, only cli option is specified
             without --json option, then it returns raw string output.
-        False otherwise
         Tuple (ret, out, err): if raw_cli_output is True
+    Raises:
+        exceptions.ExecutionError when error occurs and raw_cli_output is False
 
     Example:
         heketi_volume_create(heketi_client_node, heketi_server_url, size)
@@ -498,6 +499,7 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
     secret_arg = "--secret %s" % kwargs.get("secret") if kwargs.get("secret") else ""
     user_arg = "--user %s" % kwargs.get("user") if kwargs.get("user") else ""
 
+    err_msg = "Failed to create volume. "
     if mode == 'cli':
         cmd = ("heketi-cli -s %s volume create --size=%s %s %s %s %s %s %s "
                "%s %s %s %s %s %s %s %s %s %s"
@@ -514,8 +516,9 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
             return ret, out, err
 
         if ret != 0:
-            g.log.error("Failed to create volume using heketi")
-            return False
+            err_msg += "Out: %s \n Err: %s" % (out, err)
+            g.log.error(err_msg)
+            raise exceptions.ExecutionError(err_msg)
 
         if json_arg:
             vol_created_info = json.loads(out)
@@ -530,9 +533,9 @@ def heketi_volume_create(heketi_client_node, heketi_server_url, size,
             admin_key = kwargs.pop('secret', 'My Secret')
             conn = HeketiClient(heketi_server_url, user, admin_key)
             volume_create_info = conn.volume_create(kwargs)
-        except:
-            g.log.error("Failed to create volume using heketi")
-            return False
+        except Exception:
+            g.log.error(err_msg)
+            raise
 
         g.log.info("Volume creation is successful using heketi")
         return volume_create_info
@@ -691,30 +694,27 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
 
     Returns:
         str: volume delete command output on success
-        False on failure
         Tuple (ret, out, err): if raw_cli_output is True
+    Raises:
+        exceptions.ExecutionError when error occurs and raw_cli_output is False
 
     Example:
         heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id)
     """
 
-    (heketi_server_url,
-     json_arg, admin_key, user) = _set_heketi_global_flags(heketi_server_url,
-                                                           **kwargs)
-
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    err_msg = "Failed to delete '%s' volume. " % volume_id
     if mode == 'cli':
-        cmd = ("heketi-cli -s %s volume delete %s %s %s %s"
-               % (heketi_server_url, volume_id, json_arg, admin_key, user))
-
+        cmd = "heketi-cli -s %s volume delete %s %s %s %s" % (
+            heketi_server_url, volume_id, json_arg, admin_key, user)
         ret, out, err = g.run(heketi_client_node, cmd)
-
         if raw_cli_output:
             return ret, out, err
-
         if ret != 0:
-            g.log.error("Failed to execute heketi-cli volume delete command")
-            return False
-
+            err_msg += "Out: %s, \nErr: %s" % (out, err)
+            g.log.error(err_msg)
+            raise exceptions.ExecutionError(err_msg)
         return out
     else:
         try:
@@ -722,9 +722,9 @@ def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
             admin_key = admin_key.split('t ')[-1] if admin_key else admin_key
             conn = HeketiClient(heketi_server_url, user, admin_key)
             ret = conn.volume_delete(volume_id)
-        except:
-            g.log.error("Failed to do volume delete using heketi")
-            return False
+        except Exception:
+            g.log.error(err_msg)
+            raise
         return ret
 
 
diff --git a/tests/functional/common/heketi/heketi_tests/test_disabling_device.py b/tests/functional/common/heketi/heketi_tests/test_disabling_device.py
index 1b293f4c..41cd560c 100644
--- a/tests/functional/common/heketi/heketi_tests/test_disabling_device.py
+++ b/tests/functional/common/heketi/heketi_tests/test_disabling_device.py
@@ -1,123 +1,133 @@
-#!/usr/bin/python
-
-from __future__ import division
-import math
-
 from glusto.core import Glusto as g
-from glustolibs.gluster.exceptions import ExecutionError
 from glustolibs.gluster.volume_ops import get_volume_info
-from cnslibs.common.heketi_libs import HeketiClientSetupBaseClass
-from cnslibs.common.heketi_ops import (heketi_volume_create,
-                                       heketi_device_disable,
-                                       heketi_device_enable,
-                                       heketi_device_info,
-                                       heketi_volume_delete)
+
+from cnslibs.common import exceptions
+from cnslibs.common import heketi_libs
+from cnslibs.common import heketi_ops
+from cnslibs.common import openshift_ops
 from cnslibs.common import podcmd
-from cnslibs.common.openshift_ops import oc_rsh, get_ocp_gluster_pod_names
 
 
-class TestHeketiVolume(HeketiClientSetupBaseClass):
+class TestDisableHeketiDevice(heketi_libs.HeketiClientSetupBaseClass):
+    @podcmd.GlustoPod()
+    def test_create_volumes_enabling_and_disabling_heketi_devices(self):
+        """Test case CNS-763"""
 
-    @classmethod
-    def setUpClass(cls):
-        super(TestHeketiVolume, cls).setUpClass()
-        cls.volume_size = cls.heketi_volume['size']
+        # Get nodes info
+        node_id_list = heketi_ops.heketi_node_list(
+            self.heketi_client_node, self.heketi_server_url)
+        node_info_list = []
+        for node_id in node_id_list[0:3]:
+            node_info = heketi_ops.heketi_node_info(
+                self.heketi_client_node, self.heketi_server_url,
+                node_id, json=True)
+            node_info_list.append(node_info)
+
+        # Disable 4th and other nodes
+        if len(node_id_list) > 3:
+            for node in node_id_list[3:]:
+                heketi_ops.heketi_node_disable(
+                    self.heketi_client_node, self.heketi_server_url, node_id)
+                self.addCleanup(
+                    heketi_ops.heketi_node_enable, self.heketi_client_node,
+                    self.heketi_server_url, node_id)
+
+        # Disable second and other devices on the first 3 nodes
+        for node_info in node_info_list[0:3]:
+            devices = node_info["devices"]
+            self.assertTrue(
+                devices, "Node '%s' does not have devices." % node_info["id"])
+            if devices[0]["state"].strip().lower() != "online":
+                self.skipTest("Test expects first device to be enabled.")
+            if len(devices) < 2:
+                continue
+            for device in node_info["devices"][1:]:
+                out = heketi_ops.heketi_device_disable(
+                    self.heketi_client_node, self.heketi_server_url,
+                    device["id"])
+                self.assertTrue(
+                    out, "Failed to disable the device %s" % device["id"])
+                self.addCleanup(
+                    heketi_ops.heketi_device_enable,
+                    self.heketi_client_node, self.heketi_server_url,
+                    device["id"])
 
-    @podcmd.GlustoPod()
-    def test_to_disable_device_and_create_vol(self):
-        """
-        Disabling a device
-        """
-        size = 610
-        volume_id_list = []
         # Create heketi volume
-        g.log.info("Creating a heketi volume")
-        out = heketi_volume_create(self.heketi_client_node,
-                                   self.heketi_server_url,
-                                   self.volume_size, json=True)
-        self.assertTrue(out, ("Failed to create "
-                        "heketi volume of size %s" % str(self.volume_size)))
-        g.log.info("Successfully created heketi volume of size %s" % str(self.volume_size))
-        volume_id = out["bricks"][0]["volume"]
+        out = heketi_ops.heketi_volume_create(
+            self.heketi_client_node, self.heketi_server_url, 1, json=True)
+        self.assertTrue(out, "Failed to create heketi volume of size 1")
+        g.log.info("Successfully created heketi volume of size 1")
         device_id = out["bricks"][0]["device"]
+        self.addCleanup(self.delete_volumes, [out["bricks"][0]["volume"]])
 
         # Disable device
-        g.log.info("Disabling a device")
-        out = heketi_device_disable(self.heketi_client_node,
-                                    self.heketi_server_url,
-                                    device_id)
-        self.assertTrue(out, ("Failed to disable "
-                        "the device %s" % device_id))
+        g.log.info("Disabling '%s' device" % device_id)
+        out = heketi_ops.heketi_device_disable(
+            self.heketi_client_node, self.heketi_server_url, device_id)
+        self.assertTrue(out, "Failed to disable the device %s" % device_id)
         g.log.info("Successfully disabled device %s" % device_id)
 
-        # Get device info
-        g.log.info("Retrieving device info")
-        out = heketi_device_info(self.heketi_client_node,
-                                 self.heketi_server_url,
-                                 device_id, json=True)
-        self.assertTrue(out, ("Failed to get device info %s" % device_id))
-        g.log.info("Successfully retrieved device info %s" % device_id)
-        name = out["name"]
-        if out["state"] != "offline":
-            raise ExecutionError("Device %s is now online" % name)
-        g.log.info("Device %s is now offine" % name)
-
-        # Try to create heketi volume
-        g.log.info("Creating heketi volume:"
-                   " Expected to fail")
-        out = heketi_volume_create(self.heketi_client_node,
-                                   self.heketi_server_url,
-                                   size, json=True)
-        self.assertFalse(out, ("Successfully created "
-                         "volume of size %s, Node"
-                         " has more than one device" % str(size)))
-        g.log.info("Expected output: Failed to create a "
-                   "volume of size %s since device "
-                   "is disabled %s" % (str(size), device_id))
+        try:
+            # Get device info
+            g.log.info("Retrieving '%s' device info" % device_id)
+            out = heketi_ops.heketi_device_info(
+                self.heketi_client_node, self.heketi_server_url,
+                device_id, json=True)
+            self.assertTrue(out, "Failed to get device info %s" % device_id)
+            g.log.info("Successfully retrieved device info %s" % device_id)
+            name = out["name"]
+            if out["state"].lower().strip() != "offline":
+                raise exceptions.ExecutionError(
+                    "Device %s is not in offline state." % name)
+            g.log.info("Device %s is now offine" % name)
 
-        # Enable the device
-        g.log.info("Enable the device")
-        out = heketi_device_enable(self.heketi_client_node,
-                                   self.heketi_server_url,
-                                   device_id)
-        self.assertTrue(out, ("Failed to enable the "
-                        "device %s" % device_id))
-        g.log.info("Successfully enabled device %s" % device_id)
+            # Try to create heketi volume
+            g.log.info("Creating heketi volume: Expected to fail.")
+            try:
+                out = heketi_ops.heketi_volume_create(
+                    self.heketi_client_node, self.heketi_server_url, 1,
+                    json=True)
+            except exceptions.ExecutionError:
+                g.log.info("Volume was not created as expected.")
+            else:
+                self.addCleanup(
+                    self.delete_volumes, [out["bricks"][0]["volume"]])
+                msg = "Volume unexpectedly created. Out: %s" % out
+                assert False, msg
+        finally:
+            # Enable the device back
+            g.log.info("Enable '%s' device back." % device_id)
+            out = heketi_ops.heketi_device_enable(
+                self.heketi_client_node, self.heketi_server_url, device_id)
+            self.assertTrue(out, "Failed to enable the device %s" % device_id)
+            g.log.info("Successfully enabled device %s" % device_id)
 
         # Get device info
-        g.log.info("Retrieving device info")
-        out = heketi_device_info(self.heketi_client_node,
-                                 self.heketi_server_url,
-                                 device_id,
-                                 json=True)
+        out = heketi_ops.heketi_device_info(
+            self.heketi_client_node, self.heketi_server_url, device_id,
+            json=True)
         self.assertTrue(out, ("Failed to get device info %s" % device_id))
         g.log.info("Successfully retrieved device info %s" % device_id)
         name = out["name"]
         if out["state"] != "online":
-            raise ExecutionError("Device %s is now offline" % name)
-        g.log.info("Device %s is now online" % name)
+            raise exceptions.ExecutionError(
+                "Device %s is not in online state." % name)
 
         # Create heketi volume of size
-        g.log.info("Creating heketi volume")
-        out = heketi_volume_create(self.heketi_client_node,
-                                   self.heketi_server_url,
-                                   size, json=True)
-        self.assertTrue(out, ("Failed to create volume "
-                        "of size %s" % str(size)))
-        g.log.info("Successfully created volume of size %s" % str(size))
+        out = heketi_ops.heketi_volume_create(
+            self.heketi_client_node, self.heketi_server_url, 1, json=True)
+        self.assertTrue(out, "Failed to create volume of size 1")
+        self.addCleanup(self.delete_volumes, [out["bricks"][0]["volume"]])
+        g.log.info("Successfully created volume of size 1")
         name = out["name"]
-        volume_id = out["bricks"][0]["volume"]
-        volume_id_list.append(volume_id)
-        self.addCleanup(self.delete_volumes, volume_id_list)
 
         # Get gluster volume info
         if self.deployment_type == "cns":
-            gluster_pod = get_ocp_gluster_pod_names(
+            gluster_pod = openshift_ops.get_ocp_gluster_pod_names(
                 self.heketi_client_node)[1]
             p = podcmd.Pod(self.heketi_client_node, gluster_pod)
             out = get_volume_info(p, volname=name)
         else:
-            out = get_volume_info(self.heketi_client_node,
-                                  volname=name)
-        self.assertTrue(out, ("Failed to get volume info"))
-        g.log.info("Successfully got the volume info")
+            out = get_volume_info(self.heketi_client_node, volname=name)
+        self.assertTrue(out, "Failed to get '%s' volume info." % name)
+        g.log.info("Successfully got the '%s' volume info." % name)