Merge "Completely rewrite 'heketi_with_device_removal_insuff_space' tc"

2 files changed, 90 insertions, 169 deletions

diff --git a/tests/functional/common/heketi/test_heketi_device_operations.py b/tests/functional/common/heketi/test_heketi_device_operations.py
index 718ed80b..3fa90bc3 100644
--- a/tests/functional/common/heketi/test_heketi_device_operations.py
+++ b/tests/functional/common/heketi/test_heketi_device_operations.py
@@ -3,6 +3,7 @@ import json
 import ddt
 from glusto.core import Glusto as g
 
+from cnslibs.common.exceptions import ExecutionError
 from cnslibs.common.heketi_libs import HeketiBaseClass
 from cnslibs.common.heketi_ops import (heketi_node_enable,
                                        heketi_node_info,
@@ -319,3 +320,85 @@ class TestHeketiDeviceOperations(HeketiBaseClass):
             present,
             "Some of the '%s' volume bricks is present of the removed "
             "'%s' device." % (vol_info['id'], lowest_device_id))
+
+    def test_heketi_with_device_removal_insuff_space(self):
+        """Test case CNS-624"""
+
+        # Disable 4+ nodes and 3+ devices on the first 3 nodes
+        min_free_space_gb = 5
+        min_free_space = min_free_space_gb * 1024**2
+        heketi_url = self.heketi_server_url
+        heketi_node = self.heketi_client_node
+        nodes = {}
+
+        node_ids = heketi_node_list(heketi_node, heketi_url)
+        self.assertTrue(node_ids)
+        for node_id in node_ids:
+            node_info = heketi_node_info(
+                heketi_node, heketi_url, node_id, json=True)
+            if (node_info["state"].lower() != "online" or
+                    not node_info["devices"]):
+                continue
+            if len(nodes) > 2:
+                heketi_node_disable(heketi_node, heketi_url, node_id)
+                self.addCleanup(
+                    heketi_node_enable, heketi_node, heketi_url, node_id)
+                continue
+            for device in node_info["devices"]:
+                if device["state"].lower() != "online":
+                    continue
+                free_space = device["storage"]["free"]
+                if node_id not in nodes:
+                    nodes[node_id] = []
+                if (free_space < min_free_space or len(nodes[node_id]) > 1):
+                    heketi_device_disable(
+                        heketi_node, heketi_url, device["id"])
+                    self.addCleanup(
+                        heketi_device_enable,
+                        heketi_node, heketi_url, device["id"])
+                    continue
+                nodes[node_id].append({
+                    "device_id": device["id"], "free": free_space})
+
+        # Skip test if nodes requirements are not met
+        if (len(nodes) < 3 or
+                not all(map((lambda _list: len(_list) > 1), nodes.values()))):
+            raise self.skipTest(
+                "Could not find 3 online nodes with 2 online devices "
+                "having free space bigger than %dGb." % min_free_space_gb)
+
+        # Calculate size of a potential distributed vol
+        if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]:
+            index = 0
+        else:
+            index = 1
+        vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024 ** 2)) + 1
+        device_id = nodes[node_ids[0]][index]["device_id"]
+
+        # Create volume with such size that we consume space more than
+        # size of smaller disks
+        try:
+            heketi_vol = heketi_volume_create(
+                heketi_node, heketi_url, vol_size_gb, json=True)
+        except Exception as e:
+            g.log.warning(
+                "Got following error trying to create '%s'Gb vol: %s" % (
+                    vol_size_gb, e))
+            vol_size_gb -= 1
+            heketi_vol = heketi_volume_create(
+                heketi_node, heketi_url, vol_size_gb, json=True)
+        self.addCleanup(self.delete_volumes, heketi_vol["bricks"][0]["volume"])
+
+        # Try to 'remove' bigger Heketi disk expecting error,
+        # because there is no space on smaller disk to relocate bricks to
+        heketi_device_disable(heketi_node, heketi_url, device_id)
+        self.addCleanup(
+            heketi_device_enable, heketi_node, heketi_url, device_id)
+        try:
+            self.assertRaises(
+                ExecutionError, heketi_device_remove,
+                heketi_node, heketi_url, device_id)
+        except Exception:
+            self.addCleanup(
+                heketi_device_disable, heketi_node, heketi_url, device_id)
+            raise
diff --git a/tests/functional/common/heketi/test_heketi_volume_operations.py b/tests/functional/common/heketi/test_heketi_volume_operations.py
index 404d11cb..f82521c5 100644
--- a/tests/functional/common/heketi/test_heketi_volume_operations.py
+++ b/tests/functional/common/heketi/test_heketi_volume_operations.py
@@ -1,19 +1,8 @@
-from unittest import skip
-
-from glusto.core import Glusto as g
 from cnslibs.common.heketi_ops import (heketi_volume_delete,
                                        heketi_volume_create,
                                        heketi_volume_expand,
-                                       heketi_volume_info,
-                                       heketi_device_add,
-                                       heketi_device_enable,
-                                       heketi_device_disable,
-                                       heketi_device_remove,
-                                       heketi_device_delete,
-                                       heketi_node_info,
-                                       heketi_node_list)
+                                       heketi_volume_info)
 from cnslibs.common.heketi_libs import HeketiBaseClass
-from cnslibs.common.exceptions import ExecutionError
 
 
 class TestHeketiVolumeOperations(HeketiBaseClass):
@@ -24,58 +13,8 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
     @classmethod
     def setUpClass(cls):
         super(TestHeketiVolumeOperations, cls).setUpClass()
-        cls.volume_id = None
         cls.volume_size = 1
 
-    def volume_cleanup(self, volume_id):
-        """
-        Method to cleanup volume in self.addCleanup()
-        """
-        if volume_id is not None:
-            out = heketi_volume_delete(self.heketi_client_node,
-                                       self.heketi_server_url,
-                                       volume_id)
-            output_str = 'Volume %s deleted' % volume_id
-            if output_str not in out:
-                raise ExecutionError("Failed to delete heketi volume of"
-                                     "id %s" % volume_id)
-
-    def add_device(self, device_name, node_id):
-        """
-        Adds a device through heketi-cli
-        """
-        ret = heketi_device_add(self.heketi_client_node,
-                                self.heketi_server_url,
-                                device_name,
-                                node_id)
-
-        self.assertTrue(ret, ("Failed to add a device %s" % device_name))
-
-    def detach_devices_attached(self, device_id_list):
-        """
-        All the devices attached are gracefully
-        detached in this function
-        """
-        if not isinstance(device_id_list, (list, set, tuple)):
-            device_id_list = [device_id_list]
-
-        for device_id in device_id_list:
-            device_disable = heketi_device_disable(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_disable, False,
-                "Device %s could not be disabled" % device_id)
-            device_remove = heketi_device_remove(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_remove, False,
-                "Device %s could not be removed" % device_id)
-            device_delete = heketi_device_delete(
-                self.heketi_client_node, self.heketi_server_url, device_id)
-            self.assertNotEqual(
-                device_delete, False,
-                "Device %s could not be deleted" % device_id)
-
     def test_heketi_with_default_options(self):
         """
         Test to create volume with default options.
@@ -86,7 +25,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                                         self.volume_size, json=True)
         self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
                                    % self.volume_size))
-        self.addCleanup(self.volume_cleanup, vol_info['id'])
+        self.addCleanup(
+            heketi_volume_delete,
+            self.heketi_client_node, self.heketi_server_url, vol_info['id'])
 
         self.assertEqual(vol_info['size'], self.volume_size,
                          ("Failed to create volume with default options."
@@ -103,7 +44,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                                         self.volume_size, json=True)
         self.assertTrue(vol_info, ("Failed to create heketi volume of size %s"
                                    % self.volume_size))
-        self.addCleanup(self.volume_cleanup, vol_info['id'])
+        self.addCleanup(
+            heketi_volume_delete,
+            self.heketi_client_node, self.heketi_server_url, vol_info['id'])
         self.assertEqual(vol_info['size'], self.volume_size,
                          ("Failed to create volume."
                           "Expected Size: %s, Actual Size: %s"
@@ -123,108 +66,3 @@ class TestHeketiVolumeOperations(HeketiBaseClass):
                          ("Volume Expansion failed Expected Size: %s, Actual "
                           "Size: %s" % (str(expected_size),
                                         str(volume_info['size']))))
-
-    @skip("Blocked by BZ-1629889")
-    def test_heketi_with_device_removal_insuff_space(self):
-        """
-        Test to create volume consuming all space and then adding new device
-        and then trying to remove an existing device. We should get an error
-        saying insufficient space when removing device.
-        """
-        device_id_list = []
-
-        vol_info = heketi_volume_create(self.heketi_client_node,
-                                        self.heketi_server_url,
-                                        650,
-                                        json=True)
-
-        self.assertNotEqual(vol_info, False, "Failed to create heketi volume")
-        self.addCleanup(self.volume_cleanup, vol_info["id"])
-
-        node_id_list = heketi_node_list(
-            self.heketi_client_node, self.heketi_server_url)
-
-        for node_id in node_id_list[:2]:
-            device_present = False
-            node_info = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_id, json=True)
-
-            self.assertNotEqual(
-                node_info, False,
-                "Heketi node info on node %s failed" % node_id)
-
-            node_ip = node_info["hostnames"]["storage"][0]
-
-            for gluster_server in g.config["gluster_servers"].keys():
-                gluster_server_ip = (g.config["gluster_servers"]
-                                     [gluster_server]["storage"])
-                if gluster_server_ip == node_ip:
-                    device_name = (g.config["gluster_servers"][gluster_server]
-                                   ["additional_devices"][0])
-                    break
-            device_addition_info = heketi_device_add(
-                self.heketi_client_node, self.heketi_server_url,
-                device_name, node_id, json=True)
-
-            self.assertNotEqual(device_addition_info, False,
-                                "Device %s addition failed" % device_name)
-
-            node_info_after_addition = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_id, json=True)
-
-            self.assertNotEqual(node_info_after_addition, False,
-                                "Node info failed for node %s" % node_id)
-
-            self.assertNotEqual(
-                node_info_after_addition["devices"], [],
-                "No devices in node %s" % node_id)
-
-            for device in node_info_after_addition["devices"]:
-                if device["name"] == device_name:
-                    device_present = True
-                    device_id_list.append(device["id"])
-                    break
-
-            self.assertEqual(device_present, True,
-                             "device %s not present" % device["id"])
-
-        self.addCleanup(self.detach_devices_attached, device_id_list)
-
-        node_1_id = node_id_list[0]
-
-        node_1_info = heketi_node_info(
-                self.heketi_client_node, self.heketi_server_url,
-                node_1_id, json=True)
-
-        self.assertNotEqual(node_1_info, False,
-                            "Node info failed for node %s" % node_1_id)
-        self.assertNotEqual(
-            node_1_info["devices"], [],
-            "No devices in node %s" % node_1_id)
-        device = any([d for d in node_1_info["devices"]
-                      if device["id"] != device_id_list[0]])
-        device_disable = heketi_device_disable(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"])
-        self.assertNotEqual(
-            device_disable, False,
-            "Device %s could not be disabled" % device["id"])
-        ret, out, err = heketi_device_remove(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"],
-            raw_cli_output=True)
-        self.assertNotEqual(ret, 0, "Device %s removal successfull")
-        msg = "Error: Failed to remove device, error: No " +\
-              "Replacement was found for resource requested to be " +\
-              "removed"
-        self.assertEqual(
-            msg, err.strip(),
-            "Device %s removal failed due to invalid reason")
-        device_enable = heketi_device_enable(
-            self.heketi_client_node, self.heketi_server_url,
-            device["id"])
-        self.assertNotEqual(
-            device_enable, False,
-            "Device %s could not be enabled" % device["id"])