summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/gluster_ops.py7
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/heketi_ops.py38
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/podcmd.py12
-rw-r--r--tests/functional/heketi/test_heketi_node_operations.py279
4 files changed, 331 insertions, 5 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
index 785bde5..33ffa18 100644
--- a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
@@ -26,16 +26,17 @@ from openshiftstoragelibs import waiter
@podcmd.GlustoPod()
-def wait_to_heal_complete(timeout=300, wait_step=5):
+def wait_to_heal_complete(
+ timeout=300, wait_step=5, g_node="auto_get_gluster_endpoint"):
"""Monitors heal for volumes on gluster"""
- gluster_vol_list = get_volume_list("auto_get_gluster_endpoint")
+ gluster_vol_list = get_volume_list(g_node)
if not gluster_vol_list:
raise AssertionError("failed to get gluster volume list")
_waiter = waiter.Waiter(timeout=timeout, interval=wait_step)
for gluster_vol in gluster_vol_list:
for w in _waiter:
- if is_heal_complete("auto_get_gluster_endpoint", gluster_vol):
+ if is_heal_complete(g_node, gluster_vol):
# NOTE(vponomar): Reset attempts for waiter to avoid redundant
# sleep equal to 'interval' on the next usage.
_waiter._attempt = 0
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
index f1e535f..df00dbf 100644
--- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
@@ -1845,3 +1845,41 @@ def get_vol_file_servers_and_hosts(
glusterfs['device'].split(":")[:1]
+ glusterfs['options']['backup-volfile-servers'].split(","))
return {'vol_servers': vol_servers, 'vol_hosts': glusterfs['hosts']}
+
+
+def get_bricks_on_heketi_node(
+ heketi_client_node, heketi_server_url, node_id, **kwargs):
+ """Get bricks on heketi node.
+
+ Args:
+ heketi_client_node (str): Node on which cmd has to be executed.
+ heketi_server_url (str): Heketi server url
+ node_id (str): Node ID
+
+ Kwargs:
+ The keys, values in kwargs are:
+ - secret : (str)|None
+ - user : (str)|None
+
+ Returns:
+ list: list of bricks.
+
+ Raises:
+ AssertionError: if command fails.
+ """
+
+ if 'json' in kwargs:
+ raise AssertionError("json is not expected parameter")
+
+ kwargs['json'] = True
+
+ node_info = heketi_node_info(
+ heketi_client_node, heketi_server_url, node_id, **kwargs)
+
+ if len(node_info['devices']) < 1:
+ raise AssertionError("No device found on node %s" % node_info)
+
+ bricks = []
+ for device in node_info['devices']:
+ bricks += device['bricks']
+ return bricks
diff --git a/openshift-storage-libs/openshiftstoragelibs/podcmd.py b/openshift-storage-libs/openshiftstoragelibs/podcmd.py
index 62fff01..33e88a2 100644
--- a/openshift-storage-libs/openshiftstoragelibs/podcmd.py
+++ b/openshift-storage-libs/openshiftstoragelibs/podcmd.py
@@ -49,6 +49,7 @@ from collections import namedtuple
from functools import partial, wraps
from glusto.core import Glusto as g
+import mock
import six
from openshiftstoragelibs import openshift_ops
@@ -85,14 +86,21 @@ def run(target, command, user=None, log_level=None, orig_run=g.run):
# definition time in order to capture the method before
# any additional monkeypatching by other code
- if target == 'auto_get_gluster_endpoint':
- ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0]
+ ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0]
+ with mock.patch.object(g, 'run', new=orig_run):
gluster_pods = openshift_ops.get_ocp_gluster_pod_details(
ocp_client_node)
+
+ if target == 'auto_get_gluster_endpoint':
if gluster_pods:
target = Pod(ocp_client_node, gluster_pods[0]["pod_name"])
else:
target = list(g.config.get("gluster_servers", {}).keys())[0]
+ elif not isinstance(target, Pod) and gluster_pods:
+ for g_pod in gluster_pods:
+ if target in (g_pod['pod_host_ip'], g_pod['pod_hostname']):
+ target = Pod(ocp_client_node, g_pod['pod_name'])
+ break
if isinstance(target, Pod):
prefix = ['oc', 'rsh', target.podname]
diff --git a/tests/functional/heketi/test_heketi_node_operations.py b/tests/functional/heketi/test_heketi_node_operations.py
index 4870bc1..fd91d5c 100644
--- a/tests/functional/heketi/test_heketi_node_operations.py
+++ b/tests/functional/heketi/test_heketi_node_operations.py
@@ -1,9 +1,11 @@
+import ddt
from glusto.core import Glusto as g
from glustolibs.gluster import peer_ops
import six
from openshiftstoragelibs import baseclass
from openshiftstoragelibs import exceptions
+from openshiftstoragelibs import gluster_ops
from openshiftstoragelibs import heketi_ops
from openshiftstoragelibs import openshift_ops
from openshiftstoragelibs import openshift_storage_version
@@ -11,10 +13,17 @@ from openshiftstoragelibs import podcmd
from openshiftstoragelibs import utils
+@ddt.ddt
class TestHeketiNodeOperations(baseclass.BaseClass):
"""Class to test heketi node operations
"""
+ def setUp(self):
+ super(TestHeketiNodeOperations, self).setUp()
+ self.node = self.ocp_master_node[0]
+ self.h_node = self.heketi_client_node
+ self.h_url = self.heketi_server_url
+
@podcmd.GlustoPod()
def test_heketi_node_list(self):
"""Test node list operation
@@ -245,3 +254,273 @@ class TestHeketiNodeOperations(baseclass.BaseClass):
err_msg = ("Unexpectedly node %s got added to cluster %s" % (
storage_hostname, cluster_id))
self.assertFalse(storage_hostname, err_msg)
+
+ def get_node_to_be_added(self):
+ try:
+ # Initializes additional gluster nodes variables
+ self.additional_gluster_servers = list(
+ g.config['additional_gluster_servers'].keys())
+ self.additional_gluster_servers_info = (
+ g.config['additional_gluster_servers'])
+ return list(self.additional_gluster_servers_info.values())[0]
+ except (KeyError, AttributeError):
+ self.skipTest("Required 'additional_gluster_servers' option is "
+ "not set in the config file.")
+
+ def get_vol_ids_by_pvc_names(self, pvc_names):
+ vol_ids = []
+ custom = (r':.metadata.annotations."gluster\.kubernetes\.io\/'
+ 'heketi-volume-id"')
+ for pvc in pvc_names:
+ pv = openshift_ops.get_pv_name_from_pvc(self.node, pvc)
+ vol_id = openshift_ops.oc_get_custom_resource(
+ self.node, 'pv', custom, pv)
+ vol_ids.append(vol_id[0])
+ return vol_ids
+
+ def get_vol_names_from_vol_ids(self, vol_ids):
+ vol_names = []
+ for vol_id in vol_ids:
+ vol_info = heketi_ops.heketi_volume_info(
+ self.h_node, self.h_url, vol_id, json=True)
+ vol_names.append(vol_info['name'])
+ return vol_names
+
+ def verify_gluster_peer_status(
+ self, gluster_node, new_node_manage, new_node_storage,
+ state='present'):
+
+ # Verify gluster peer status
+ peer_status = openshift_ops.cmd_run_on_gluster_pod_or_node(
+ self.node, 'gluster peer status', gluster_node)
+ found = (new_node_manage in peer_status
+ or new_node_storage in peer_status)
+ if state == 'present':
+ msg = ('Node %s did not get attached in gluster peer status %s'
+ % (new_node_manage, peer_status))
+ self.assertTrue(found, msg)
+ elif state == 'absent':
+ msg = ('Node %s did not get deattached in gluster peer status %s'
+ % (new_node_manage, peer_status))
+ self.assertFalse(found, msg)
+ else:
+ msg = "State %s is other than present, absent" % state
+ raise AssertionError(msg)
+
+ def verify_node_is_present_or_not_in_heketi(
+ self, node_id, manage_hostname, storage_ip, state='present'):
+
+ topology = heketi_ops.heketi_topology_info(
+ self.h_node, self.h_url, json=True)
+ if state == 'present':
+ present = False
+ for node in topology['clusters'][0]['nodes']:
+ if node_id == node['id']:
+ self.assertEqual(
+ manage_hostname, node['hostnames']['manage'][0])
+ self.assertEqual(
+ storage_ip, node['hostnames']['storage'][0])
+ present = True
+ break
+ self.assertTrue(present, 'Node %s not found in heketi' % node_id)
+
+ elif state == 'absent':
+ for node in topology['clusters'][0]['nodes']:
+ self.assertNotEqual(
+ manage_hostname, node['hostnames']['manage'][0])
+ self.assertNotEqual(
+ storage_ip, node['hostnames']['storage'][0])
+ self.assertNotEqual(node_id, node['id'])
+ else:
+ msg = "State %s is other than present, absent" % state
+ raise AssertionError(msg)
+
+ def verify_gluster_server_present_in_heketi_vol_info_or_not(
+ self, vol_ids, gluster_server, state='present'):
+
+ # Verify gluster servers in vol info
+ for vol_id in vol_ids:
+ g_servers = heketi_ops.get_vol_file_servers_and_hosts(
+ self.h_node, self.h_url, vol_id)
+ g_servers = (g_servers['vol_servers'] + g_servers['vol_hosts'])
+ if state == 'present':
+ self.assertIn(gluster_server, g_servers)
+ elif state == 'absent':
+ self.assertNotIn(gluster_server, g_servers)
+ else:
+ msg = "State %s is other than present, absent" % state
+ raise AssertionError(msg)
+
+ def verify_volume_bricks_are_present_or_not_on_heketi_node(
+ self, vol_ids, node_id, state='present'):
+
+ for vol_id in vol_ids:
+ vol_info = heketi_ops.heketi_volume_info(
+ self.h_node, self.h_url, vol_id, json=True)
+ bricks = vol_info['bricks']
+ self.assertFalse((len(bricks) % 3))
+ if state == 'present':
+ found = False
+ for brick in bricks:
+ if brick['node'] == node_id:
+ found = True
+ break
+ self.assertTrue(
+ found, 'Bricks of vol %s does not present on node %s'
+ % (vol_id, node_id))
+ elif state == 'absent':
+ for brick in bricks:
+ self.assertNotEqual(
+ brick['node'], node_id, 'Bricks of vol %s is present '
+ 'on node %s' % (vol_id, node_id))
+ else:
+ msg = "State %s is other than present, absent" % state
+ raise AssertionError(msg)
+
+ def get_ready_for_node_add(self, hostname):
+ self.configure_node_to_run_gluster(hostname)
+
+ h_nodes = heketi_ops.heketi_node_list(self.h_node, self.h_url)
+
+ # Disable nodes except first two nodes
+ for node_id in h_nodes[2:]:
+ heketi_ops.heketi_node_disable(self.h_node, self.h_url, node_id)
+ self.addCleanup(
+ heketi_ops.heketi_node_enable, self.h_node, self.h_url,
+ node_id)
+
+ def add_device_on_heketi_node(self, node_id, device_name):
+
+ # Add Devices on heketi node
+ heketi_ops.heketi_device_add(
+ self.h_node, self.h_url, device_name, node_id)
+
+ # Get device id of newly added device
+ node_info = heketi_ops.heketi_node_info(
+ self.h_node, self.h_url, node_id, json=True)
+ for device in node_info['devices']:
+ if device['name'] == device_name:
+ return device['id']
+ raise AssertionError('Device %s did not found on node %s' % (
+ device_name, node_id))
+
+ def delete_node_and_devices_on_it(self, node_id):
+
+ heketi_ops.heketi_node_disable(self.h_node, self.h_url, node_id)
+ heketi_ops.heketi_node_remove(self.h_node, self.h_url, node_id)
+ node_info = heketi_ops.heketi_node_info(
+ self.h_node, self.h_url, node_id, json=True)
+ for device in node_info['devices']:
+ heketi_ops.heketi_device_delete(
+ self.h_node, self.h_url, device['id'])
+ heketi_ops.heketi_node_delete(self.h_node, self.h_url, node_id)
+
+ @ddt.data('remove', 'delete')
+ def test_heketi_node_remove_or_delete(self, operation='delete'):
+ """Test node remove and delete functionality of heketi and validate
+ gluster peer status and heketi topology.
+ """
+ # Get node info to be added in heketi cluster
+ new_node = self.get_node_to_be_added()
+ new_node_manage = new_node['manage']
+ new_node_storage = new_node['storage']
+
+ self.get_ready_for_node_add(new_node_manage)
+
+ h, h_node, h_url = heketi_ops, self.h_node, self.h_url
+
+ # Get cluster id where node needs to be added.
+ cluster_id = h.heketi_cluster_list(h_node, h_url, json=True)
+ cluster_id = cluster_id['clusters'][0]
+
+ h_nodes_list = h.heketi_node_list(h_node, h_url)
+
+ node_needs_cleanup = False
+ try:
+ # Add new node
+ h_new_node = h.heketi_node_add(
+ h_node, h_url, 1, cluster_id, new_node_manage,
+ new_node_storage, json=True)['id']
+ node_needs_cleanup = True
+
+ # Get hostname of first gluster node
+ g_node = h.heketi_node_info(
+ h_node, h_url, h_nodes_list[0],
+ json=True)['hostnames']['manage'][0]
+
+ # Verify gluster peer status
+ self.verify_gluster_peer_status(
+ g_node, new_node_manage, new_node_storage)
+
+ # Add Devices on newly added node
+ device_id = self.add_device_on_heketi_node(
+ h_new_node, new_node['devices'][0])
+
+ # Create PVC's and DC's
+ vol_count = 5
+ pvcs = self.create_and_wait_for_pvcs(pvc_amount=vol_count)
+ dcs = self.create_dcs_with_pvc(pvcs)
+
+ # Get vol id's
+ vol_ids = self.get_vol_ids_by_pvc_names(pvcs)
+
+ # Get bricks count on newly added node
+ bricks = h.get_bricks_on_heketi_node(
+ h_node, h_url, h_new_node)
+ self.assertGreaterEqual(len(bricks), vol_count)
+
+ # Enable the nodes back, which were disabled earlier
+ for node_id in h_nodes_list[2:]:
+ h.heketi_node_enable(h_node, h_url, node_id)
+
+ if operation == 'remove':
+ # Remove the node
+ h.heketi_node_disable(h_node, h_url, h_new_node)
+ h.heketi_node_remove(h_node, h_url, h_new_node)
+ # Delete the device
+ h.heketi_device_delete(h_node, h_url, device_id)
+ elif operation == 'delete':
+ # Remove and delete device
+ h.heketi_device_disable(h_node, h_url, device_id)
+ h.heketi_device_remove(h_node, h_url, device_id)
+ h.heketi_device_delete(h_node, h_url, device_id)
+ # Remove node
+ h.heketi_node_disable(h_node, h_url, h_new_node)
+ h.heketi_node_remove(h_node, h_url, h_new_node)
+ else:
+ msg = "Operation %s is other than remove, delete." % operation
+ raise AssertionError(msg)
+
+ # Delete Node
+ h.heketi_node_delete(h_node, h_url, h_new_node)
+ node_needs_cleanup = False
+
+ # Verify node is deleted from heketi
+ self.verify_node_is_present_or_not_in_heketi(
+ h_new_node, new_node_manage, new_node_storage, state='absent')
+
+ # Verify gluster peer status
+ self.verify_gluster_peer_status(
+ g_node, new_node_manage, new_node_storage, state='absent')
+
+ # Verify gluster servers are not present in vol info
+ self.verify_gluster_server_present_in_heketi_vol_info_or_not(
+ vol_ids, new_node_storage, state='absent')
+
+ # Verify vol bricks are not present on deleted nodes
+ self.verify_volume_bricks_are_present_or_not_on_heketi_node(
+ vol_ids, new_node_storage, state='absent')
+
+ # Wait for heal to complete
+ gluster_ops.wait_to_heal_complete(g_node=g_node)
+
+ for _, pod in dcs.values():
+ openshift_ops.wait_for_pod_be_ready(self.node, pod, timeout=1)
+ finally:
+ # Cleanup newly added Node
+ if node_needs_cleanup:
+ self.addCleanup(self.delete_node_and_devices_on_it, h_new_node)
+
+ # Enable the nodes back, which were disabled earlier
+ for node_id in h_nodes_list[2:]:
+ self.addCleanup(h.heketi_node_enable, h_node, h_url, node_id)