diff options
Diffstat (limited to 'tests')
4 files changed, 155 insertions, 400 deletions
diff --git a/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py b/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py index 7963413b..af9d2b44 100644 --- a/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py +++ b/tests/functional/common/heketi/heketi_tests/test_heketi_create_volume.py @@ -1,5 +1,3 @@ -import time - from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import get_volume_list, get_volume_info import six @@ -206,33 +204,31 @@ class TestHeketiVolume(HeketiBaseClass): heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol1['id']) - # Sleep for couple of seconds to avoid races - time.sleep(2) - - # Get info about block hosting volume available space + # Get info about block hosting volumes file_volumes = heketi_volume_list( self.heketi_client_node, self.heketi_server_url, json=True) self.assertTrue(file_volumes) - max_freesize = 0 - file_volumes_debug_info = [] + self.assertIn("volumes", file_volumes) + self.assertTrue(file_volumes["volumes"]) + max_block_hosting_vol_size, file_volumes_debug_info = 0, [] for vol_id in file_volumes["volumes"]: vol = heketi_volume_info( self.heketi_client_node, self.heketi_server_url, vol_id, json=True) - current_freesize = vol.get("blockinfo", {}).get("freesize", 0) - if current_freesize > max_freesize: - max_freesize = current_freesize - if current_freesize: + current_block_hosting_vol_size = vol.get('size', 0) + if current_block_hosting_vol_size > max_block_hosting_vol_size: + max_block_hosting_vol_size = current_block_hosting_vol_size + if current_block_hosting_vol_size: file_volumes_debug_info.append(six.text_type({ 'id': vol.get('id', '?'), 'name': vol.get('name', '?'), - 'size': vol.get('size', '?'), + 'size': current_block_hosting_vol_size, 'blockinfo': vol.get('blockinfo', '?'), })) - self.assertGreater(max_freesize, 0) + self.assertGreater(max_block_hosting_vol_size, 0) # Try to create blockvolume with size bigger than available - too_big_vol_size = max_freesize + 1 + too_big_vol_size = max_block_hosting_vol_size + 1 try: blockvol2 = heketi_blockvolume_create( self.heketi_client_node, self.heketi_server_url, @@ -244,9 +240,14 @@ class TestHeketiVolume(HeketiBaseClass): self.addCleanup( heketi_blockvolume_delete, self.heketi_client_node, self.heketi_server_url, blockvol2['id']) - self.assertFalse( - blockvol2, - "Volume unexpectedly was created. Calculated 'max free size' is " - "'%s'.\nBlock volume info is: %s \n" - "Block hosting volumes which were considered: \n%s" % ( - max_freesize, blockvol2, '\n'.join(file_volumes_debug_info))) + block_hosting_vol = heketi_volume_info( + self.heketi_client_node, self.heketi_server_url, + blockvol2.get('blockhostingvolume'), json=True) + self.assertGreater( + block_hosting_vol.get('size', -2), blockvol2.get('size', -1), + ("Block volume unexpectedly was created. " + "Calculated 'max free size' is '%s'.\nBlock volume info is: %s \n" + "File volume info, which hosts block volume: \n%s," + "Block hosting volumes which were considered: \n%s" % ( + max_block_hosting_vol_size, blockvol2, block_hosting_vol, + '\n'.join(file_volumes_debug_info)))) diff --git a/tests/functional/common/heketi/test_heketi_device_operations.py b/tests/functional/common/heketi/test_heketi_device_operations.py index 202da90f..3fa90bc3 100644 --- a/tests/functional/common/heketi/test_heketi_device_operations.py +++ b/tests/functional/common/heketi/test_heketi_device_operations.py @@ -1,7 +1,9 @@ import json +import ddt from glusto.core import Glusto as g +from cnslibs.common.exceptions import ExecutionError from cnslibs.common.heketi_libs import HeketiBaseClass from cnslibs.common.heketi_ops import (heketi_node_enable, heketi_node_info, @@ -17,6 +19,7 @@ from cnslibs.common.heketi_ops import (heketi_node_enable, heketi_topology_info) +@ddt.ddt class TestHeketiDeviceOperations(HeketiBaseClass): """Test Heketi device enable/disable and remove functionality.""" @@ -168,8 +171,10 @@ class TestHeketiDeviceOperations(HeketiBaseClass): "None of '%s' volume bricks is present on the '%s' device." % ( vol_info['id'], online_device_id)) - def test_device_remove_operation(self): - """Test case CNS-766. Test device remove functionality.""" + @ddt.data(True, False) + def test_device_remove_operation(self, delete_device): + """Test cases CNS-623,766.""" + gluster_server_0 = g.config["gluster_servers"].values()[0] try: device_name = gluster_server_0["additional_devices"][0] @@ -206,6 +211,7 @@ class TestHeketiDeviceOperations(HeketiBaseClass): device["storage"]["total"] < lowest_device_size): lowest_device_size = device["storage"]["total"] lowest_device_id = device["id"] + lowest_device_name = device["name"] if lowest_device_id is None: self.skipTest( "Didn't find suitable device for disablement on '%s' node." % ( @@ -260,15 +266,41 @@ class TestHeketiDeviceOperations(HeketiBaseClass): # Need to disable device before removing heketi_device_disable( - self.heketi_client_node, self.heketi_server_url, lowest_device_id) - self.addCleanup(heketi_device_enable, self.heketi_client_node, - self.heketi_server_url, lowest_device_id) + self.heketi_client_node, self.heketi_server_url, + lowest_device_id) + if not delete_device: + self.addCleanup(heketi_device_enable, self.heketi_client_node, + self.heketi_server_url, lowest_device_id) # Remove device from Heketi - heketi_device_remove( - self.heketi_client_node, self.heketi_server_url, lowest_device_id) - self.addCleanup(heketi_device_disable, self.heketi_client_node, - self.heketi_server_url, lowest_device_id) + try: + heketi_device_remove( + self.heketi_client_node, self.heketi_server_url, + lowest_device_id) + except Exception: + if delete_device: + self.addCleanup(heketi_device_enable, self.heketi_client_node, + self.heketi_server_url, lowest_device_id) + raise + if not delete_device: + self.addCleanup(heketi_device_disable, self.heketi_client_node, + self.heketi_server_url, lowest_device_id) + + if delete_device: + try: + heketi_device_delete( + self.heketi_client_node, self.heketi_server_url, + lowest_device_id) + except Exception: + self.addCleanup(heketi_device_enable, self.heketi_client_node, + self.heketi_server_url, lowest_device_id) + self.addCleanup(heketi_device_disable, self.heketi_client_node, + self.heketi_server_url, lowest_device_id) + raise + self.addCleanup( + heketi_device_add, + self.heketi_client_node, self.heketi_server_url, + lowest_device_name, node_id) # Create volume vol_info = heketi_volume_create(self.heketi_client_node, @@ -278,6 +310,9 @@ class TestHeketiDeviceOperations(HeketiBaseClass): "Failed to create heketi volume of size %d" % vol_size)) self.addCleanup(self.delete_volumes, vol_info['id']) + if delete_device: + return + # Check that none of volume's bricks is present on the device present = self.check_any_of_bricks_present_in_device( vol_info['bricks'], lowest_device_id) @@ -285,3 +320,85 @@ class TestHeketiDeviceOperations(HeketiBaseClass): present, "Some of the '%s' volume bricks is present of the removed " "'%s' device." % (vol_info['id'], lowest_device_id)) + + def test_heketi_with_device_removal_insuff_space(self): + """Test case CNS-624""" + + # Disable 4+ nodes and 3+ devices on the first 3 nodes + min_free_space_gb = 5 + min_free_space = min_free_space_gb * 1024**2 + heketi_url = self.heketi_server_url + heketi_node = self.heketi_client_node + nodes = {} + + node_ids = heketi_node_list(heketi_node, heketi_url) + self.assertTrue(node_ids) + for node_id in node_ids: + node_info = heketi_node_info( + heketi_node, heketi_url, node_id, json=True) + if (node_info["state"].lower() != "online" or + not node_info["devices"]): + continue + if len(nodes) > 2: + heketi_node_disable(heketi_node, heketi_url, node_id) + self.addCleanup( + heketi_node_enable, heketi_node, heketi_url, node_id) + continue + for device in node_info["devices"]: + if device["state"].lower() != "online": + continue + free_space = device["storage"]["free"] + if node_id not in nodes: + nodes[node_id] = [] + if (free_space < min_free_space or len(nodes[node_id]) > 1): + heketi_device_disable( + heketi_node, heketi_url, device["id"]) + self.addCleanup( + heketi_device_enable, + heketi_node, heketi_url, device["id"]) + continue + nodes[node_id].append({ + "device_id": device["id"], "free": free_space}) + + # Skip test if nodes requirements are not met + if (len(nodes) < 3 or + not all(map((lambda _list: len(_list) > 1), nodes.values()))): + raise self.skipTest( + "Could not find 3 online nodes with 2 online devices " + "having free space bigger than %dGb." % min_free_space_gb) + + # Calculate size of a potential distributed vol + if nodes[node_ids[0]][0]["free"] > nodes[node_ids[0]][1]["free"]: + index = 0 + else: + index = 1 + vol_size_gb = int(nodes[node_ids[0]][index]["free"] / (1024 ** 2)) + 1 + device_id = nodes[node_ids[0]][index]["device_id"] + + # Create volume with such size that we consume space more than + # size of smaller disks + try: + heketi_vol = heketi_volume_create( + heketi_node, heketi_url, vol_size_gb, json=True) + except Exception as e: + g.log.warning( + "Got following error trying to create '%s'Gb vol: %s" % ( + vol_size_gb, e)) + vol_size_gb -= 1 + heketi_vol = heketi_volume_create( + heketi_node, heketi_url, vol_size_gb, json=True) + self.addCleanup(self.delete_volumes, heketi_vol["bricks"][0]["volume"]) + + # Try to 'remove' bigger Heketi disk expecting error, + # because there is no space on smaller disk to relocate bricks to + heketi_device_disable(heketi_node, heketi_url, device_id) + self.addCleanup( + heketi_device_enable, heketi_node, heketi_url, device_id) + try: + self.assertRaises( + ExecutionError, heketi_device_remove, + heketi_node, heketi_url, device_id) + except Exception: + self.addCleanup( + heketi_device_disable, heketi_node, heketi_url, device_id) + raise diff --git a/tests/functional/common/heketi/test_heketi_volume_operations.py b/tests/functional/common/heketi/test_heketi_volume_operations.py index ac7df139..f82521c5 100644 --- a/tests/functional/common/heketi/test_heketi_volume_operations.py +++ b/tests/functional/common/heketi/test_heketi_volume_operations.py @@ -1,20 +1,8 @@ -from unittest import skip - -from glusto.core import Glusto as g from cnslibs.common.heketi_ops import (heketi_volume_delete, heketi_volume_create, heketi_volume_expand, - heketi_volume_info, - heketi_topology_info, - heketi_device_add, - heketi_device_enable, - heketi_device_disable, - heketi_device_remove, - heketi_device_delete, - heketi_node_info, - heketi_node_list) + heketi_volume_info) from cnslibs.common.heketi_libs import HeketiBaseClass -from cnslibs.common.exceptions import ExecutionError class TestHeketiVolumeOperations(HeketiBaseClass): @@ -25,58 +13,8 @@ class TestHeketiVolumeOperations(HeketiBaseClass): @classmethod def setUpClass(cls): super(TestHeketiVolumeOperations, cls).setUpClass() - cls.volume_id = None cls.volume_size = 1 - def volume_cleanup(self, volume_id): - """ - Method to cleanup volume in self.addCleanup() - """ - if volume_id is not None: - out = heketi_volume_delete(self.heketi_client_node, - self.heketi_server_url, - volume_id) - output_str = 'Volume %s deleted' % volume_id - if output_str not in out: - raise ExecutionError("Failed to delete heketi volume of" - "id %s" % volume_id) - - def add_device(self, device_name, node_id): - """ - Adds a device through heketi-cli - """ - ret = heketi_device_add(self.heketi_client_node, - self.heketi_server_url, - device_name, - node_id) - - self.assertTrue(ret, ("Failed to add a device %s" % device_name)) - - def detach_devices_attached(self, device_id_list): - """ - All the devices attached are gracefully - detached in this function - """ - if not isinstance(device_id_list, (list, set, tuple)): - device_id_list = [device_id_list] - - for device_id in device_id_list: - device_disable = heketi_device_disable( - self.heketi_client_node, self.heketi_server_url, device_id) - self.assertNotEqual( - device_disable, False, - "Device %s could not be disabled" % device_id) - device_remove = heketi_device_remove( - self.heketi_client_node, self.heketi_server_url, device_id) - self.assertNotEqual( - device_remove, False, - "Device %s could not be removed" % device_id) - device_delete = heketi_device_delete( - self.heketi_client_node, self.heketi_server_url, device_id) - self.assertNotEqual( - device_delete, False, - "Device %s could not be deleted" % device_id) - def test_heketi_with_default_options(self): """ Test to create volume with default options. @@ -87,7 +25,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass): self.volume_size, json=True) self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) - self.addCleanup(self.volume_cleanup, vol_info['id']) + self.addCleanup( + heketi_volume_delete, + self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume with default options." @@ -104,7 +44,9 @@ class TestHeketiVolumeOperations(HeketiBaseClass): self.volume_size, json=True) self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" % self.volume_size)) - self.addCleanup(self.volume_cleanup, vol_info['id']) + self.addCleanup( + heketi_volume_delete, + self.heketi_client_node, self.heketi_server_url, vol_info['id']) self.assertEqual(vol_info['size'], self.volume_size, ("Failed to create volume." "Expected Size: %s, Actual Size: %s" @@ -124,203 +66,3 @@ class TestHeketiVolumeOperations(HeketiBaseClass): ("Volume Expansion failed Expected Size: %s, Actual " "Size: %s" % (str(expected_size), str(volume_info['size'])))) - - @skip("Blocked by BZ-1629889") - def test_heketi_with_device_removal_insuff_space(self): - """ - Test to create volume consuming all space and then adding new device - and then trying to remove an existing device. We should get an error - saying insufficient space when removing device. - """ - device_id_list = [] - - vol_info = heketi_volume_create(self.heketi_client_node, - self.heketi_server_url, - 650, - json=True) - - self.assertNotEqual(vol_info, False, "Failed to create heketi volume") - self.addCleanup(self.volume_cleanup, vol_info["id"]) - - node_id_list = heketi_node_list( - self.heketi_client_node, self.heketi_server_url) - - for node_id in node_id_list[:2]: - device_present = False - node_info = heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_id, json=True) - - self.assertNotEqual( - node_info, False, - "Heketi node info on node %s failed" % node_id) - - node_ip = node_info["hostnames"]["storage"][0] - - for gluster_server in g.config["gluster_servers"].keys(): - gluster_server_ip = (g.config["gluster_servers"] - [gluster_server]["storage"]) - if gluster_server_ip == node_ip: - device_name = (g.config["gluster_servers"][gluster_server] - ["additional_devices"][0]) - break - device_addition_info = heketi_device_add( - self.heketi_client_node, self.heketi_server_url, - device_name, node_id, json=True) - - self.assertNotEqual(device_addition_info, False, - "Device %s addition failed" % device_name) - - node_info_after_addition = heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_id, json=True) - - self.assertNotEqual(node_info_after_addition, False, - "Node info failed for node %s" % node_id) - - self.assertNotEqual( - node_info_after_addition["devices"], [], - "No devices in node %s" % node_id) - - for device in node_info_after_addition["devices"]: - if device["name"] == device_name: - device_present = True - device_id_list.append(device["id"]) - break - - self.assertEqual(device_present, True, - "device %s not present" % device["id"]) - - self.addCleanup(self.detach_devices_attached, device_id_list) - - node_1_id = node_id_list[0] - - node_1_info = heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_1_id, json=True) - - self.assertNotEqual(node_1_info, False, - "Node info failed for node %s" % node_1_id) - self.assertNotEqual( - node_1_info["devices"], [], - "No devices in node %s" % node_1_id) - device = any([d for d in node_1_info["devices"] - if device["id"] != device_id_list[0]]) - device_disable = heketi_device_disable( - self.heketi_client_node, self.heketi_server_url, - device["id"]) - self.assertNotEqual( - device_disable, False, - "Device %s could not be disabled" % device["id"]) - ret, out, err = heketi_device_remove( - self.heketi_client_node, self.heketi_server_url, - device["id"], - raw_cli_output=True) - self.assertNotEqual(ret, 0, "Device %s removal successfull") - msg = "Error: Failed to remove device, error: No " +\ - "Replacement was found for resource requested to be " +\ - "removed" - self.assertEqual( - msg, err.strip(), - "Device %s removal failed due to invalid reason") - device_enable = heketi_device_enable( - self.heketi_client_node, self.heketi_server_url, - device["id"]) - self.assertNotEqual( - device_enable, False, - "Device %s could not be enabled" % device["id"]) - - @skip("Blocked by BZ-1629889") - def test_device_remove_basic_validation(self): - """ - Test to create volume after a device removal and with new device added. - """ - - vol_info = heketi_volume_create(self.heketi_client_node, - self.heketi_server_url, - self.volume_size, json=True) - self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" - % self.volume_size)) - self.addCleanup(self.volume_cleanup, vol_info['id']) - - self.assertEqual(vol_info['size'], self.volume_size, - ("Failed to create volume with default options." - "Expected Size: %s, Actual Size: %s" - % (self.volume_size, vol_info['size']))) - - # 1. Device addition - gluster_srvrs = self.gluster_servers - - device_name = (g.config["gluster_servers"][gluster_srvrs[0]] - ["additional_devices"][0]) - manage_hostname = (g.config["gluster_servers"] - [gluster_srvrs[0]]["manage"]) - # Now, get node id of corresponding hostname - topo_info = heketi_topology_info(self.heketi_client_node, - self.heketi_server_url, - json=True) - - self.assertNotEqual( - topo_info["clusters"][0]["nodes"], [], - "Nodes don't exist, empty cluster") - - node_id = None - for node in topo_info["clusters"][0]["nodes"]: - if manage_hostname == node['hostnames']["manage"][0]: - node_id = node["id"] - break - self.assertNotEqual( - node_id, None, - "No information about node_id for %s" % manage_hostname) - self.add_device(device_name, node_id) - - # get other device id for deletion - topo_info = heketi_topology_info(self.heketi_client_node, - self.heketi_server_url, - json=True) - - self.assertNotEqual( - topo_info["clusters"][0]["nodes"], [], - "No information about nodes") - - device_id_flag1 = False - device_id_flag2 = False - - for node in topo_info["clusters"][0]["nodes"]: - if node["id"] != node_id: - continue - self.assertNotEqual( - node["devices"], [], - "Device list empty for node %s" % node_id) - for device in node["devices"]: - device_id = device["id"] - if device_name != device["name"]: - device_id_flag1 = True - else: - self.addCleanup(self.detach_devices_attached, device_id) - device_id_flag2 = True - if device_id_flag1 and device_id_flag2: - break - break - - self.detach_devices_attached(device_id) - - # Check whether deleted device removed from topology info. - topo_info = heketi_topology_info(self.heketi_client_node, - self.heketi_server_url, - json=True) - match = False - for device in topo_info["clusters"][0]["nodes"][0]["devices"]: - if device_id == device["id"]: - match = True - break - self.assertFalse(match, "Device:%s still present in topology" - % device_id) - - # Volume creation after device update - vol_info = heketi_volume_create(self.heketi_client_node, - self.heketi_server_url, - self.volume_size, json=True) - self.assertTrue(vol_info, ("Failed to create heketi volume of size %s" - % self.volume_size)) - self.addCleanup(self.volume_cleanup, vol_info['id']) diff --git a/tests/functional/common/heketi/test_volume_expansion_and_devices.py b/tests/functional/common/heketi/test_volume_expansion_and_devices.py index 262e3d6b..05f39700 100644 --- a/tests/functional/common/heketi/test_volume_expansion_and_devices.py +++ b/tests/functional/common/heketi/test_volume_expansion_and_devices.py @@ -1,7 +1,5 @@ from __future__ import division -import json import math -import unittest from glusto.core import Glusto as g from glustolibs.gluster import volume_ops, rebalance_ops @@ -161,109 +159,6 @@ class TestVolumeExpansionAndDevicesTestCases(HeketiBaseClass): device_delete, False, "Device %s could not be deleted" % device_id) - @unittest.skip("Blocked by BZ-1629889") - @podcmd.GlustoPod() - def test_add_device_heketi_cli(self): - """ - Method to test heketi device addition with background - gluster validation - """ - device_id_list = [] - hosts = [] - gluster_servers = [] - - node_id_list = heketi_ops.heketi_node_list( - self.heketi_client_node, self.heketi_server_url) - - creation_info = heketi_ops.heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, 100, json=True) - - self.assertNotEqual(creation_info, False, - "Volume creation failed") - - self.addCleanup(self.delete_volumes, creation_info["id"]) - - ret, out, err = heketi_ops.heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, 620, json=True, - raw_cli_output=True) - - self.assertEqual(ret, 255, "Volume creation did not fail ret- %s " - "out- %s err= %s" % (ret, out, err)) - g.log.info("Volume creation failed as expected, err- %s" % err) - - if ret == 0: - out_json = json.loads(out) - self.addCleanup(self.delete_volumes, out_json["id"]) - - for node_id in node_id_list: - device_present = False - node_info = heketi_ops.heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_id, json=True) - - self.assertNotEqual( - node_info, False, - "Heketi node info on node %s failed" % node_id) - - node_ip = node_info["hostnames"]["storage"][0] - - for gluster_server in g.config["gluster_servers"].keys(): - gluster_server_ip = (g.config["gluster_servers"] - [gluster_server]["storage"]) - if gluster_server_ip == node_ip: - device_name = (g.config["gluster_servers"][gluster_server] - ["additional_devices"][0]) - break - device_addition_info = heketi_ops.heketi_device_add( - self.heketi_client_node, self.heketi_server_url, - device_name, node_id, json=True) - - self.assertNotEqual(device_addition_info, False, - "Device %s addition failed" % device_name) - - node_info_after_addition = heketi_ops.heketi_node_info( - self.heketi_client_node, self.heketi_server_url, - node_id, json=True) - for device in node_info_after_addition["devices"]: - if device["name"] == device_name: - device_present = True - device_id_list.append(device["id"]) - - self.assertEqual(device_present, True, - "device %s not present" % device["id"]) - - self.addCleanup(self.detach_devices_attached, device_id_list) - - output_dict = heketi_ops.heketi_volume_create( - self.heketi_client_node, self.heketi_server_url, - 620, json=True) - - self.assertNotEqual(output_dict, False, "Volume creation failed") - self.addCleanup(self.delete_volumes, output_dict["id"]) - - self.assertEqual(output_dict["durability"]["replicate"]["replica"], 3) - self.assertEqual(output_dict["size"], 620) - mount_node = (output_dict["mount"]["glusterfs"] - ["device"].strip().split(":")[0]) - - hosts.append(mount_node) - backup_volfile_server_list = ( - output_dict["mount"]["glusterfs"]["options"] - ["backup-volfile-servers"].strip().split(",")) - - for backup_volfile_server in backup_volfile_server_list: - hosts.append(backup_volfile_server) - for gluster_server in g.config["gluster_servers"].keys(): - gluster_servers.append(g.config["gluster_servers"] - [gluster_server]["storage"]) - self.assertEqual( - set(hosts), set(gluster_servers), - "Hosts do not match gluster servers for %s" % output_dict["id"]) - - volume_name = output_dict["name"] - - self.get_brick_and_volume_status(volume_name) - def test_volume_expansion_expanded_volume(self): """ To test volume expansion with brick and rebalance |