summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArun Kumar <arukumar@redhat.com>2019-05-30 18:49:18 +0530
committervponomar <vponomar@redhat.com>2019-09-25 12:49:03 +0000
commitc3181ad2210ad76a1f27be046b43b108a71dc4f1 (patch)
tree6382f03ac3bffcaf764e01d41ea4dd964cd75765
parent203410a740e275cce78e840c30c583c82e563c28 (diff)
Add TC Fail tcmu-runner service while creating and deleting pvc
Create 8 Block PVC and wait for them to get bound. Delete created PVC's and create 8 more PVC's simultaneously. Kill the tcmu-runner service while PVC creation and deletion is in the progress. Restart all the services related to block volumes and verify all pending operations are completed sucessfully. Change-Id: I0cb4cd29b92233a65be93f4b96f1a9a0cb8bed9f
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/openshift_ops.py30
-rw-r--r--tests/functional/gluster_stability/test_gluster_block_stability.py89
2 files changed, 119 insertions, 0 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
index 62c6affe..07c90d7a 100644
--- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
@@ -34,6 +34,9 @@ PODS_WIDE_RE = re.compile(
SERVICE_STATUS = "systemctl status %s"
SERVICE_RESTART = "systemctl restart %s"
SERVICE_STATUS_REGEX = r"Active: (.*) \((.*)\) since .*;.*"
+PGREP_SERVICE = "pgrep %s"
+KILL_SERVICE = "kill -9 %s"
+IS_ACTIVE_SERVICE = "systemctl is-active %s"
def oc_get_pods(ocp_node, selector=None):
@@ -1487,6 +1490,33 @@ def restart_service_on_gluster_pod_or_node(ocp_client, service, gluster_node):
ocp_client, SERVICE_RESTART % service, gluster_node)
+def kill_service_on_gluster_pod_or_node(ocp_client, service, gluster_node):
+ """Kill service on Gluster pod or node.
+
+ Args:
+ ocp_client (str): host on which we want to run 'oc' commands.
+ service (str): service which needs to be killed
+ gluster_node (str): Gluster node IPv4 which stores either Gluster POD
+ or Gluster services directly.
+ Raises:
+ AssertionError: In case killing of a service fails.
+ """
+ # Get the pid of the service which needs to be killed
+ get_service_pids = cmd_run_on_gluster_pod_or_node(
+ ocp_client, PGREP_SERVICE % service, gluster_node=gluster_node)
+ service_pids = " ".join(get_service_pids.split("\n"))
+ cmd_run_on_gluster_pod_or_node(
+ ocp_client, KILL_SERVICE % service_pids, gluster_node=gluster_node)
+
+ # Check the status of service
+ try:
+ cmd_run_on_gluster_pod_or_node(
+ ocp_client, IS_ACTIVE_SERVICE % service, gluster_node=gluster_node)
+ except exceptions.ExecutionError as err:
+ if "command terminated with exit code 3" not in str(err):
+ raise
+
+
def oc_adm_manage_node(
ocp_client, operation, nodes=None, node_selector=None):
"""Manage common operations on nodes for administrators.
diff --git a/tests/functional/gluster_stability/test_gluster_block_stability.py b/tests/functional/gluster_stability/test_gluster_block_stability.py
index 163d9712..33ac3509 100644
--- a/tests/functional/gluster_stability/test_gluster_block_stability.py
+++ b/tests/functional/gluster_stability/test_gluster_block_stability.py
@@ -27,6 +27,8 @@ from openshiftstoragelibs.openshift_ops import (
get_ocp_gluster_pod_details,
get_pod_name_from_dc,
get_pv_name_from_pvc,
+ get_pvc_status,
+ kill_service_on_gluster_pod_or_node,
oc_adm_manage_node,
oc_create_pvc,
oc_delete,
@@ -34,12 +36,15 @@ from openshiftstoragelibs.openshift_ops import (
oc_get_pv,
oc_get_schedulable_nodes,
oc_rsh,
+ restart_service_on_gluster_pod_or_node,
scale_dcs_pod_amount_and_wait,
verify_pvc_status_is_bound,
wait_for_events,
wait_for_ocp_node_be_ready,
wait_for_pod_be_ready,
+ wait_for_pvcs_be_bound,
wait_for_resource_absence,
+ wait_for_resources_absence,
wait_for_service_status_on_gluster_pod_or_node,
)
from openshiftstoragelibs.openshift_storage_libs import (
@@ -873,3 +878,87 @@ class TestGlusterBlockStability(GlusterBlockBaseClass):
msg = ("Expected size %s of file '%s' is not present "
"in out '%s'" % (file_size, _file, out))
self.assertIn(six.text_type(file_size), out, msg)
+
+ def test_tcmu_runner_failure_while_creating_and_deleting_pvc(self):
+ """Kill the tcmu-runner service while creating and deleting PVC's"""
+
+ self.prefix = "auto-block-test-%s" % utils.get_random_str()
+
+ # Create DC and pod with created PVC.
+ sc_name = self.create_storage_class(
+ hacount=len(self.gluster_servers), vol_name_prefix=self.prefix)
+ pvc_name = self.create_and_wait_for_pvc(sc_name=sc_name)
+ dc_name, pod_name = self.create_dc_with_pvc(pvc_name)
+
+ # Validate iscsi and multipath
+ self.verify_iscsi_sessions_and_multipath(pvc_name, dc_name)
+
+ # Create 8 PVC's and add to list
+ pvc_names_for_creations, pv_names_for_deletions = [], []
+ pvc_names_for_deletions = self.create_and_wait_for_pvcs(
+ pvc_amount=8, sc_name=sc_name)
+
+ # Get their pv names and add to list
+ for pvc_name in pvc_names_for_deletions:
+ pv_names_for_deletions.append(
+ get_pv_name_from_pvc(self.node, pvc_name))
+
+ # Delete PVC's without wait which were created earlier
+ for delete_pvc_name in pvc_names_for_deletions:
+ oc_delete(self.node, 'pvc', delete_pvc_name)
+
+ # Create PVC's without wait at the same time and add to list
+ create_pvc_name = oc_create_pvc(self.node, sc_name=sc_name)
+ pvc_names_for_creations.append(create_pvc_name)
+ self.addCleanup(
+ wait_for_resource_absence, self.node, 'pvc', create_pvc_name)
+ pvc_names = " ".join(pvc_names_for_creations)
+ self.addCleanup(oc_delete, self.node, "pvc", pvc_names)
+
+ # Kill Tcmu-runner service
+ services = ("tcmu-runner", "gluster-block-target", "gluster-blockd")
+ kill_service_on_gluster_pod_or_node(
+ self.node, "tcmu-runner", self.gluster_servers[0])
+ self.addCleanup(
+ cmd_run_on_gluster_pod_or_node,
+ self.node, "systemctl restart %s" % " ".join(services),
+ gluster_node=self.gluster_servers[0])
+
+ # Get the names of PV's which are in Released state after PVC deletion
+ pending_creations, pending_deletions = [], []
+ for pv_name in pv_names_for_deletions:
+ # Check "Released" status for PV and add it to the list
+ try:
+ pv_status = oc_get_custom_resource(
+ self.node, "pv", ":.status.phase", pv_name)[0]
+ if pv_status == 'Released':
+ pending_deletions.append(pv_name)
+ except AssertionError:
+ pass
+
+ # Check which PVC's are in pending state and add to pending list
+ for pvc_name_in_creations in pvc_names_for_creations:
+ if get_pvc_status(self.node, pvc_name_in_creations) == 'Pending':
+ pending_creations.append(pvc_name_in_creations)
+
+ # Restart the services
+ for service in services:
+ state = (
+ 'exited' if service == 'gluster-block-target' else 'running')
+ restart_service_on_gluster_pod_or_node(
+ self.node, service, self.gluster_servers[0])
+ wait_for_service_status_on_gluster_pod_or_node(
+ self.node, service, 'active', state, self.gluster_servers[0])
+
+ # Wait for PV's absence and PVC's getting bound
+ wait_for_resources_absence(self.node, 'pv', pending_deletions)
+ wait_for_pvcs_be_bound(self.node, pending_creations, timeout=300)
+
+ # Validate volumes in heketi blockvolume list
+ heketi_volumes = heketi_blockvolume_list(
+ self.heketi_client_node, self.heketi_server_url)
+ volume_count = heketi_volumes.count(self.prefix)
+ msg = (
+ 'Wrong volume count in heketi blockvolume list %s and expected '
+ 'volume count is 9 ' % volume_count)
+ self.assertEqual(9, volume_count, msg)