summaryrefslogtreecommitdiffstats
path: root/tests/functional/gluster_stability/test_gluster_block_stability.py
diff options
context:
space:
mode:
authorArun Kumar <arukumar@arukumar.remote.csb>2019-09-16 15:17:49 +0530
committerArun <arukumar@redhat.com>2019-11-26 18:42:01 +0530
commitf51ac495904dcd43649dbdc9a3997593aa8f9164 (patch)
treec4221031a71f45cb1c474b9675f5d7eb74491426 /tests/functional/gluster_stability/test_gluster_block_stability.py
parent1ed3d8401476b56e1f84cfdb9eb849f07cc80af8 (diff)
Add TC deleting bunch of PVC's during network failure
Create network failure while deleting PVC's Network side failure is introduced by opening and closing the ports related to gluster-blockd. Change-Id: Id3a749aa1a051bbce99b85046fa0a79831e85dd5
Diffstat (limited to 'tests/functional/gluster_stability/test_gluster_block_stability.py')
-rw-r--r--tests/functional/gluster_stability/test_gluster_block_stability.py80
1 files changed, 80 insertions, 0 deletions
diff --git a/tests/functional/gluster_stability/test_gluster_block_stability.py b/tests/functional/gluster_stability/test_gluster_block_stability.py
index e81949f..59852e9 100644
--- a/tests/functional/gluster_stability/test_gluster_block_stability.py
+++ b/tests/functional/gluster_stability/test_gluster_block_stability.py
@@ -37,6 +37,7 @@ from openshiftstoragelibs.openshift_ops import (
get_ocp_gluster_pod_details,
get_pod_name_from_dc,
get_pv_name_from_pvc,
+ get_vol_names_from_pv,
kill_service_on_gluster_pod_or_node,
match_pv_and_heketi_block_volumes,
oc_adm_manage_node,
@@ -1245,3 +1246,82 @@ class TestGlusterBlockStability(GlusterBlockBaseClass):
self.verify_all_paths_are_up_in_multipath(
list(mpaths)[0], hacount, ini_node, timeout=1)
+
+ def get_vol_id_and_vol_names_from_pvc_names(self, pvc_names):
+ vol_details = []
+ for pvc_name in pvc_names:
+ pv_name = get_pv_name_from_pvc(self.node, pvc_name)
+ volume = get_vol_names_from_pv(
+ self.node, pv_name, vol_type='block')
+ vol_details.append(volume)
+ return vol_details
+
+ def check_errors_in_heketi_pod_network_failure_after_deletion(
+ self, since_time, vol_names):
+ # Get name of heketi pod
+ heketi_pod_name = get_pod_name_from_dc(self.node, self.heketi_dc_name)
+
+ # Check for errors in heketi pod logs
+ err_cmd = (
+ r'oc logs %s --since-time="%s" | grep " Failed to run command '
+ r'\[gluster-block delete*"' % (heketi_pod_name, since_time))
+
+ for w in Waiter(30, 5):
+ err_out = cmd_run(err_cmd, self.node)
+ if any(vol_name in err_out for vol_name in vol_names):
+ break
+ if w.expired:
+ err_msg = ("Expected ERROR for volumes '%s' not generated in "
+ "heketi pod logs" % vol_names)
+ raise AssertionError(err_msg)
+
+ def test_delete_block_pvcs_with_network_failure(self):
+ """Block port 24010 while deleting PVC's"""
+ pvc_amount, pvc_delete_amount = 10, 5
+ gluster_node = self.gluster_servers[0]
+ chain = 'OS_FIREWALL_ALLOW'
+ rules = '-p tcp -m state --state NEW -m tcp --dport 24010 -j ACCEPT'
+
+ sc_name = self.create_storage_class(hacount=len(self.gluster_servers))
+
+ # Get the total free space available
+ initial_free_storage = get_total_free_space(
+ self.heketi_client_node, self.heketi_server_url)
+
+ # Create 10 PVC's, get their PV names and volume ids
+ pvc_names = self.create_and_wait_for_pvcs(
+ sc_name=sc_name, pvc_amount=pvc_amount, timeout=240)
+ vol_details = self.get_vol_id_and_vol_names_from_pvc_names(pvc_names)
+ vol_names = [vol_name['gluster_vol'] for vol_name in vol_details]
+
+ # Delete 5 PVCs not waiting for the results
+ oc_delete(self.node, 'pvc', " ".join(pvc_names[:pvc_delete_amount]))
+
+ # Get time to collect logs
+ since_time = cmd_run(
+ 'date -u --rfc-3339=ns| cut -d "+" -f 1', self.node).replace(
+ " ", "T") + "Z"
+
+ try:
+ # Close the port #24010 on gluster node and then delete other 5 PVC
+ # without wait
+ node_delete_iptables_rules(gluster_node, chain, rules)
+ oc_delete(
+ self.node, 'pvc', ' '.join(pvc_names[pvc_delete_amount:]))
+ self.addCleanup(
+ wait_for_resources_absence,
+ self.node, 'pvc', pvc_names[pvc_delete_amount:])
+
+ # Check errors in heketi pod logs
+ self.check_errors_in_heketi_pod_network_failure_after_deletion(
+ since_time, vol_names[pvc_delete_amount:])
+ finally:
+ # Open port 24010
+ node_add_iptables_rules(gluster_node, chain, rules)
+
+ # validate available free space is same
+ final_free_storage = get_total_free_space(
+ self.heketi_client_node, self.heketi_server_url)
+ msg = ("Available free space %s is not same as expected %s"
+ % (initial_free_storage, final_free_storage))
+ self.assertEqual(initial_free_storage, final_free_storage, msg)