diff options
4 files changed, 331 insertions, 5 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py index 785bde58..33ffa18d 100644 --- a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py @@ -26,16 +26,17 @@ from openshiftstoragelibs import waiter @podcmd.GlustoPod() -def wait_to_heal_complete(timeout=300, wait_step=5): +def wait_to_heal_complete( + timeout=300, wait_step=5, g_node="auto_get_gluster_endpoint"): """Monitors heal for volumes on gluster""" - gluster_vol_list = get_volume_list("auto_get_gluster_endpoint") + gluster_vol_list = get_volume_list(g_node) if not gluster_vol_list: raise AssertionError("failed to get gluster volume list") _waiter = waiter.Waiter(timeout=timeout, interval=wait_step) for gluster_vol in gluster_vol_list: for w in _waiter: - if is_heal_complete("auto_get_gluster_endpoint", gluster_vol): + if is_heal_complete(g_node, gluster_vol): # NOTE(vponomar): Reset attempts for waiter to avoid redundant # sleep equal to 'interval' on the next usage. _waiter._attempt = 0 diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py index f1e535fa..df00dbf3 100644 --- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py @@ -1845,3 +1845,41 @@ def get_vol_file_servers_and_hosts( glusterfs['device'].split(":")[:1] + glusterfs['options']['backup-volfile-servers'].split(",")) return {'vol_servers': vol_servers, 'vol_hosts': glusterfs['hosts']} + + +def get_bricks_on_heketi_node( + heketi_client_node, heketi_server_url, node_id, **kwargs): + """Get bricks on heketi node. + + Args: + heketi_client_node (str): Node on which cmd has to be executed. + heketi_server_url (str): Heketi server url + node_id (str): Node ID + + Kwargs: + The keys, values in kwargs are: + - secret : (str)|None + - user : (str)|None + + Returns: + list: list of bricks. + + Raises: + AssertionError: if command fails. + """ + + if 'json' in kwargs: + raise AssertionError("json is not expected parameter") + + kwargs['json'] = True + + node_info = heketi_node_info( + heketi_client_node, heketi_server_url, node_id, **kwargs) + + if len(node_info['devices']) < 1: + raise AssertionError("No device found on node %s" % node_info) + + bricks = [] + for device in node_info['devices']: + bricks += device['bricks'] + return bricks diff --git a/openshift-storage-libs/openshiftstoragelibs/podcmd.py b/openshift-storage-libs/openshiftstoragelibs/podcmd.py index 62fff01a..33e88a26 100644 --- a/openshift-storage-libs/openshiftstoragelibs/podcmd.py +++ b/openshift-storage-libs/openshiftstoragelibs/podcmd.py @@ -49,6 +49,7 @@ from collections import namedtuple from functools import partial, wraps from glusto.core import Glusto as g +import mock import six from openshiftstoragelibs import openshift_ops @@ -85,14 +86,21 @@ def run(target, command, user=None, log_level=None, orig_run=g.run): # definition time in order to capture the method before # any additional monkeypatching by other code - if target == 'auto_get_gluster_endpoint': - ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0] + ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0] + with mock.patch.object(g, 'run', new=orig_run): gluster_pods = openshift_ops.get_ocp_gluster_pod_details( ocp_client_node) + + if target == 'auto_get_gluster_endpoint': if gluster_pods: target = Pod(ocp_client_node, gluster_pods[0]["pod_name"]) else: target = list(g.config.get("gluster_servers", {}).keys())[0] + elif not isinstance(target, Pod) and gluster_pods: + for g_pod in gluster_pods: + if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']): + target = Pod(ocp_client_node, g_pod['pod_name']) + break if isinstance(target, Pod): prefix = ['oc', 'rsh', target.podname] diff --git a/tests/functional/heketi/test_heketi_node_operations.py b/tests/functional/heketi/test_heketi_node_operations.py index 4870bc12..fd91d5ce 100644 --- a/tests/functional/heketi/test_heketi_node_operations.py +++ b/tests/functional/heketi/test_heketi_node_operations.py @@ -1,9 +1,11 @@ +import ddt from glusto.core import Glusto as g from glustolibs.gluster import peer_ops import six from openshiftstoragelibs import baseclass from openshiftstoragelibs import exceptions +from openshiftstoragelibs import gluster_ops from openshiftstoragelibs import heketi_ops from openshiftstoragelibs import openshift_ops from openshiftstoragelibs import openshift_storage_version @@ -11,10 +13,17 @@ from openshiftstoragelibs import podcmd from openshiftstoragelibs import utils +@ddt.ddt class TestHeketiNodeOperations(baseclass.BaseClass): """Class to test heketi node operations """ + def setUp(self): + super(TestHeketiNodeOperations, self).setUp() + self.node = self.ocp_master_node[0] + self.h_node = self.heketi_client_node + self.h_url = self.heketi_server_url + @podcmd.GlustoPod() def test_heketi_node_list(self): """Test node list operation @@ -245,3 +254,273 @@ class TestHeketiNodeOperations(baseclass.BaseClass): err_msg = ("Unexpectedly node %s got added to cluster %s" % ( storage_hostname, cluster_id)) self.assertFalse(storage_hostname, err_msg) + + def get_node_to_be_added(self): + try: + # Initializes additional gluster nodes variables + self.additional_gluster_servers = list( + g.config['additional_gluster_servers'].keys()) + self.additional_gluster_servers_info = ( + g.config['additional_gluster_servers']) + return list(self.additional_gluster_servers_info.values())[0] + except (KeyError, AttributeError): + self.skipTest("Required 'additional_gluster_servers' option is " + "not set in the config file.") + + def get_vol_ids_by_pvc_names(self, pvc_names): + vol_ids = [] + custom = (r':.metadata.annotations."gluster\.kubernetes\.io\/' + 'heketi-volume-id"') + for pvc in pvc_names: + pv = openshift_ops.get_pv_name_from_pvc(self.node, pvc) + vol_id = openshift_ops.oc_get_custom_resource( + self.node, 'pv', custom, pv) + vol_ids.append(vol_id[0]) + return vol_ids + + def get_vol_names_from_vol_ids(self, vol_ids): + vol_names = [] + for vol_id in vol_ids: + vol_info = heketi_ops.heketi_volume_info( + self.h_node, self.h_url, vol_id, json=True) + vol_names.append(vol_info['name']) + return vol_names + + def verify_gluster_peer_status( + self, gluster_node, new_node_manage, new_node_storage, + state='present'): + + # Verify gluster peer status + peer_status = openshift_ops.cmd_run_on_gluster_pod_or_node( + self.node, 'gluster peer status', gluster_node) + found = (new_node_manage in peer_status + or new_node_storage in peer_status) + if state == 'present': + msg = ('Node %s did not get attached in gluster peer status %s' + % (new_node_manage, peer_status)) + self.assertTrue(found, msg) + elif state == 'absent': + msg = ('Node %s did not get deattached in gluster peer status %s' + % (new_node_manage, peer_status)) + self.assertFalse(found, msg) + else: + msg = "State %s is other than present, absent" % state + raise AssertionError(msg) + + def verify_node_is_present_or_not_in_heketi( + self, node_id, manage_hostname, storage_ip, state='present'): + + topology = heketi_ops.heketi_topology_info( + self.h_node, self.h_url, json=True) + if state == 'present': + present = False + for node in topology['clusters'][0]['nodes']: + if node_id == node['id']: + self.assertEqual( + manage_hostname, node['hostnames']['manage'][0]) + self.assertEqual( + storage_ip, node['hostnames']['storage'][0]) + present = True + break + self.assertTrue(present, 'Node %s not found in heketi' % node_id) + + elif state == 'absent': + for node in topology['clusters'][0]['nodes']: + self.assertNotEqual( + manage_hostname, node['hostnames']['manage'][0]) + self.assertNotEqual( + storage_ip, node['hostnames']['storage'][0]) + self.assertNotEqual(node_id, node['id']) + else: + msg = "State %s is other than present, absent" % state + raise AssertionError(msg) + + def verify_gluster_server_present_in_heketi_vol_info_or_not( + self, vol_ids, gluster_server, state='present'): + + # Verify gluster servers in vol info + for vol_id in vol_ids: + g_servers = heketi_ops.get_vol_file_servers_and_hosts( + self.h_node, self.h_url, vol_id) + g_servers = (g_servers['vol_servers'] + g_servers['vol_hosts']) + if state == 'present': + self.assertIn(gluster_server, g_servers) + elif state == 'absent': + self.assertNotIn(gluster_server, g_servers) + else: + msg = "State %s is other than present, absent" % state + raise AssertionError(msg) + + def verify_volume_bricks_are_present_or_not_on_heketi_node( + self, vol_ids, node_id, state='present'): + + for vol_id in vol_ids: + vol_info = heketi_ops.heketi_volume_info( + self.h_node, self.h_url, vol_id, json=True) + bricks = vol_info['bricks'] + self.assertFalse((len(bricks) % 3)) + if state == 'present': + found = False + for brick in bricks: + if brick['node'] == node_id: + found = True + break + self.assertTrue( + found, 'Bricks of vol %s does not present on node %s' + % (vol_id, node_id)) + elif state == 'absent': + for brick in bricks: + self.assertNotEqual( + brick['node'], node_id, 'Bricks of vol %s is present ' + 'on node %s' % (vol_id, node_id)) + else: + msg = "State %s is other than present, absent" % state + raise AssertionError(msg) + + def get_ready_for_node_add(self, hostname): + self.configure_node_to_run_gluster(hostname) + + h_nodes = heketi_ops.heketi_node_list(self.h_node, self.h_url) + + # Disable nodes except first two nodes + for node_id in h_nodes[2:]: + heketi_ops.heketi_node_disable(self.h_node, self.h_url, node_id) + self.addCleanup( + heketi_ops.heketi_node_enable, self.h_node, self.h_url, + node_id) + + def add_device_on_heketi_node(self, node_id, device_name): + + # Add Devices on heketi node + heketi_ops.heketi_device_add( + self.h_node, self.h_url, device_name, node_id) + + # Get device id of newly added device + node_info = heketi_ops.heketi_node_info( + self.h_node, self.h_url, node_id, json=True) + for device in node_info['devices']: + if device['name'] == device_name: + return device['id'] + raise AssertionError('Device %s did not found on node %s' % ( + device_name, node_id)) + + def delete_node_and_devices_on_it(self, node_id): + + heketi_ops.heketi_node_disable(self.h_node, self.h_url, node_id) + heketi_ops.heketi_node_remove(self.h_node, self.h_url, node_id) + node_info = heketi_ops.heketi_node_info( + self.h_node, self.h_url, node_id, json=True) + for device in node_info['devices']: + heketi_ops.heketi_device_delete( + self.h_node, self.h_url, device['id']) + heketi_ops.heketi_node_delete(self.h_node, self.h_url, node_id) + + @ddt.data('remove', 'delete') + def test_heketi_node_remove_or_delete(self, operation='delete'): + """Test node remove and delete functionality of heketi and validate + gluster peer status and heketi topology. + """ + # Get node info to be added in heketi cluster + new_node = self.get_node_to_be_added() + new_node_manage = new_node['manage'] + new_node_storage = new_node['storage'] + + self.get_ready_for_node_add(new_node_manage) + + h, h_node, h_url = heketi_ops, self.h_node, self.h_url + + # Get cluster id where node needs to be added. + cluster_id = h.heketi_cluster_list(h_node, h_url, json=True) + cluster_id = cluster_id['clusters'][0] + + h_nodes_list = h.heketi_node_list(h_node, h_url) + + node_needs_cleanup = False + try: + # Add new node + h_new_node = h.heketi_node_add( + h_node, h_url, 1, cluster_id, new_node_manage, + new_node_storage, json=True)['id'] + node_needs_cleanup = True + + # Get hostname of first gluster node + g_node = h.heketi_node_info( + h_node, h_url, h_nodes_list[0], + json=True)['hostnames']['manage'][0] + + # Verify gluster peer status + self.verify_gluster_peer_status( + g_node, new_node_manage, new_node_storage) + + # Add Devices on newly added node + device_id = self.add_device_on_heketi_node( + h_new_node, new_node['devices'][0]) + + # Create PVC's and DC's + vol_count = 5 + pvcs = self.create_and_wait_for_pvcs(pvc_amount=vol_count) + dcs = self.create_dcs_with_pvc(pvcs) + + # Get vol id's + vol_ids = self.get_vol_ids_by_pvc_names(pvcs) + + # Get bricks count on newly added node + bricks = h.get_bricks_on_heketi_node( + h_node, h_url, h_new_node) + self.assertGreaterEqual(len(bricks), vol_count) + + # Enable the nodes back, which were disabled earlier + for node_id in h_nodes_list[2:]: + h.heketi_node_enable(h_node, h_url, node_id) + + if operation == 'remove': + # Remove the node + h.heketi_node_disable(h_node, h_url, h_new_node) + h.heketi_node_remove(h_node, h_url, h_new_node) + # Delete the device + h.heketi_device_delete(h_node, h_url, device_id) + elif operation == 'delete': + # Remove and delete device + h.heketi_device_disable(h_node, h_url, device_id) + h.heketi_device_remove(h_node, h_url, device_id) + h.heketi_device_delete(h_node, h_url, device_id) + # Remove node + h.heketi_node_disable(h_node, h_url, h_new_node) + h.heketi_node_remove(h_node, h_url, h_new_node) + else: + msg = "Operation %s is other than remove, delete." % operation + raise AssertionError(msg) + + # Delete Node + h.heketi_node_delete(h_node, h_url, h_new_node) + node_needs_cleanup = False + + # Verify node is deleted from heketi + self.verify_node_is_present_or_not_in_heketi( + h_new_node, new_node_manage, new_node_storage, state='absent') + + # Verify gluster peer status + self.verify_gluster_peer_status( + g_node, new_node_manage, new_node_storage, state='absent') + + # Verify gluster servers are not present in vol info + self.verify_gluster_server_present_in_heketi_vol_info_or_not( + vol_ids, new_node_storage, state='absent') + + # Verify vol bricks are not present on deleted nodes + self.verify_volume_bricks_are_present_or_not_on_heketi_node( + vol_ids, new_node_storage, state='absent') + + # Wait for heal to complete + gluster_ops.wait_to_heal_complete(g_node=g_node) + + for _, pod in dcs.values(): + openshift_ops.wait_for_pod_be_ready(self.node, pod, timeout=1) + finally: + # Cleanup newly added Node + if node_needs_cleanup: + self.addCleanup(self.delete_node_and_devices_on_it, h_new_node) + + # Enable the nodes back, which were disabled earlier + for node_id in h_nodes_list[2:]: + self.addCleanup(h.heketi_node_enable, h_node, h_url, node_id) |