summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/openshift_ops.py16
-rw-r--r--tests/functional/gluster_stability/test_gluster_block_stability.py80
2 files changed, 91 insertions, 5 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
index 56b0629..4e668d6 100644
--- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
@@ -1358,7 +1358,7 @@ def get_pv_name_from_pvc(hostname, pvc_name):
return pv_name
-def get_vol_names_from_pv(hostname, pv_name):
+def get_vol_names_from_pv(hostname, pv_name, vol_type='file'):
'''
Returns the heketi and gluster
vol names of the corresponding PV
@@ -1368,6 +1368,7 @@ def get_vol_names_from_pv(hostname, pv_name):
pv_name (str): pv_name for which we
want to find corresponding
vol names
+ vol_type (str): volume type block or file
Returns:
volname (dict): dict if successful
{"heketi_vol": heketi_vol_name,
@@ -1377,10 +1378,15 @@ def get_vol_names_from_pv(hostname, pv_name):
otherwise raise Exception
'''
vol_dict = {}
- cmd = (r"oc get pv %s -o=custom-columns="
- r":.metadata.annotations."
- r"'gluster\.kubernetes\.io\/heketi\-volume\-id',"
- r":.spec.glusterfs.path" % pv_name)
+ if vol_type == 'block':
+ cmd = (r"oc get pv %s -o=custom-columns="
+ r":.metadata.annotations.'gluster\.org\/volume-id',"
+ r":.metadata.annotations.glusterBlockShare" % pv_name)
+ else:
+ cmd = (r"oc get pv %s -o=custom-columns="
+ r":.metadata.annotations."
+ r"'gluster\.kubernetes\.io\/heketi\-volume\-id',"
+ r":.spec.glusterfs.path" % pv_name)
vol_list = command.cmd_run(cmd, hostname=hostname).split()
vol_dict = {"heketi_vol": vol_list[0],
"gluster_vol": vol_list[1]}
diff --git a/tests/functional/gluster_stability/test_gluster_block_stability.py b/tests/functional/gluster_stability/test_gluster_block_stability.py
index e81949f..59852e9 100644
--- a/tests/functional/gluster_stability/test_gluster_block_stability.py
+++ b/tests/functional/gluster_stability/test_gluster_block_stability.py
@@ -37,6 +37,7 @@ from openshiftstoragelibs.openshift_ops import (
get_ocp_gluster_pod_details,
get_pod_name_from_dc,
get_pv_name_from_pvc,
+ get_vol_names_from_pv,
kill_service_on_gluster_pod_or_node,
match_pv_and_heketi_block_volumes,
oc_adm_manage_node,
@@ -1245,3 +1246,82 @@ class TestGlusterBlockStability(GlusterBlockBaseClass):
self.verify_all_paths_are_up_in_multipath(
list(mpaths)[0], hacount, ini_node, timeout=1)
+
+ def get_vol_id_and_vol_names_from_pvc_names(self, pvc_names):
+ vol_details = []
+ for pvc_name in pvc_names:
+ pv_name = get_pv_name_from_pvc(self.node, pvc_name)
+ volume = get_vol_names_from_pv(
+ self.node, pv_name, vol_type='block')
+ vol_details.append(volume)
+ return vol_details
+
+ def check_errors_in_heketi_pod_network_failure_after_deletion(
+ self, since_time, vol_names):
+ # Get name of heketi pod
+ heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name)
+
+ # Check for errors in heketi pod logs
+ err_cmd = (
+ r'oc logs %s --since-time="%s" | grep " Failed to run command '
+ r'\[gluster-block delete*"' % (heketi_pod_name, since_time))
+
+ for w in Waiter(30, 5):
+ err_out = cmd_run(err_cmd, self.node)
+ if any(vol_name in err_out for vol_name in vol_names):
+ break
+ if w.expired:
+ err_msg = ("Expected ERROR for volumes '%s' not generated in "
+ "heketi pod logs" % vol_names)
+ raise AssertionError(err_msg)
+
+ def test_delete_block_pvcs_with_network_failure(self):
+ """Block port 24010 while deleting PVC's"""
+ pvc_amount, pvc_delete_amount = 10, 5
+ gluster_node = self.gluster_servers[0]
+ chain = 'OS_FIREWALL_ALLOW'
+ rules = '-p tcp -m state --state NEW -m tcp --dport 24010 -j ACCEPT'
+
+ sc_name = self.create_storage_class(hacount=len(self.gluster_servers))
+
+ # Get the total free space available
+ initial_free_storage = get_total_free_space(
+ self.heketi_client_node, self.heketi_server_url)
+
+ # Create 10 PVC's, get their PV names and volume ids
+ pvc_names = self.create_and_wait_for_pvcs(
+ sc_name=sc_name, pvc_amount=pvc_amount, timeout=240)
+ vol_details = self.get_vol_id_and_vol_names_from_pvc_names(pvc_names)
+ vol_names = [vol_name['gluster_vol'] for vol_name in vol_details]
+
+ # Delete 5 PVCs not waiting for the results
+ oc_delete(self.node, 'pvc', " ".join(pvc_names[:pvc_delete_amount]))
+
+ # Get time to collect logs
+ since_time = cmd_run(
+ 'date -u --rfc-3339=ns| cut -d "+" -f 1', self.node).replace(
+ " ", "T") + "Z"
+
+ try:
+ # Close the port #24010 on gluster node and then delete other 5 PVC
+ # without wait
+ node_delete_iptables_rules(gluster_node, chain, rules)
+ oc_delete(
+ self.node, 'pvc', ' '.join(pvc_names[pvc_delete_amount:]))
+ self.addCleanup(
+ wait_for_resources_absence,
+ self.node, 'pvc', pvc_names[pvc_delete_amount:])
+
+ # Check errors in heketi pod logs
+ self.check_errors_in_heketi_pod_network_failure_after_deletion(
+ since_time, vol_names[pvc_delete_amount:])
+ finally:
+ # Open port 24010
+ node_add_iptables_rules(gluster_node, chain, rules)
+
+ # validate available free space is same
+ final_free_storage = get_total_free_space(
+ self.heketi_client_node, self.heketi_server_url)
+ msg = ("Available free space %s is not same as expected %s"
+ % (initial_free_storage, final_free_storage))
+ self.assertEqual(initial_free_storage, final_free_storage, msg)