summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/openshift_ops.py26
-rw-r--r--tests/functional/gluster_stability/test_gluster_block_stability.py99
2 files changed, 125 insertions, 0 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
index 8a94e10..60748aa 100644
--- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
@@ -1529,3 +1529,29 @@ def get_default_block_hosting_volume_size(hostname, heketi_dc_name):
"out['glusterfs']['block_hosting_volume_size'] from out:\n" % out)
g.log.error(msg)
raise exceptions.ExecutionError(msg)
+
+
+def wait_for_ocp_node_be_ready(
+ hostname, node_name, timeout=300, wait_step=5):
+ """Wait for OCP node to be in ready state.
+
+ Args:
+ hostname (str): Node where we want to run our commands.
+ node_name (str): Name of ocp node.
+ timeout (int): Seconds to wait for Node to be Ready.
+ wait_step (int): Interval in seconds to wait before checking
+ status again.
+ Raises:
+ exceptions.ExecutionError.
+ """
+ custom = r'":.status.conditions[?(@.type==\"Ready\")]".status'
+
+ for w in waiter.Waiter(timeout, wait_step):
+ status = oc_get_custom_resource(hostname, 'node', custom, node_name)
+ if status[0] == 'True':
+ return
+
+ err_msg = ("Exceeded timeout: %s for waiting for Node: %s "
+ "to be in Ready state" % (timeout, node_name))
+ g.log.error(err_msg)
+ raise exceptions.ExecutionError(err_msg)
diff --git a/tests/functional/gluster_stability/test_gluster_block_stability.py b/tests/functional/gluster_stability/test_gluster_block_stability.py
index 7c51506..1b5bf74 100644
--- a/tests/functional/gluster_stability/test_gluster_block_stability.py
+++ b/tests/functional/gluster_stability/test_gluster_block_stability.py
@@ -1,3 +1,5 @@
+import six
+
from openshiftstoragelibs.baseclass import GlusterBlockBaseClass
from openshiftstoragelibs.command import cmd_run
from openshiftstoragelibs.exceptions import ConfigError
@@ -7,6 +9,7 @@ from openshiftstoragelibs.heketi_ops import (
)
from openshiftstoragelibs.node_ops import (
find_vm_name_by_ip_or_hostname,
+ node_reboot_by_command,
power_off_vm_by_name,
power_on_vm_by_name,
)
@@ -21,6 +24,7 @@ from openshiftstoragelibs.openshift_ops import (
oc_get_schedulable_nodes,
oc_rsh,
scale_dcs_pod_amount_and_wait,
+ wait_for_ocp_node_be_ready,
wait_for_pod_be_ready,
wait_for_resource_absence,
wait_for_service_status_on_gluster_pod_or_node,
@@ -34,6 +38,9 @@ from openshiftstoragelibs.openshift_storage_libs import (
from openshiftstoragelibs.openshift_storage_version import (
get_openshift_storage_version
)
+from openshiftstoragelibs.openshift_version import (
+ get_openshift_version
+)
from openshiftstoragelibs.waiter import Waiter
@@ -447,3 +454,95 @@ class TestGlusterBlockStability(GlusterBlockBaseClass):
# Verify block sessions
self.verify_iscsi_sessions_and_multipath(
pvc, pvc_and_dc[pvc]['dc_name'])
+
+ def get_initiator_node_and_mark_other_nodes_unschedulable(self):
+
+ ocp_version = get_openshift_version(self.node)
+ if ocp_version < '3.10':
+ self.skipTest(
+ "Block functionality doesn't work in OCP less than 3.10")
+
+ # Get all nodes
+ all_nodes = oc_get_custom_resource(
+ self.node, 'no', ':.metadata.name,:.spec.unschedulable')
+ all_nodes = {node[0]: node[1] for node in all_nodes}
+
+ # Get master nodes
+ master_nodes = oc_get_custom_resource(
+ self.node, 'node', ':.metadata.name',
+ selector='node-role.kubernetes.io/master=true')
+ master_nodes = [node[0] for node in master_nodes]
+
+ # Get list of all gluster nodes
+ g_nodes = oc_get_custom_resource(
+ self.node, 'pod', ':.spec.nodeName', selector='glusterfs-node=pod')
+ g_nodes = [node[0] for node in g_nodes]
+
+ # Get the list of nodes other than gluster and master
+ o_nodes = list(
+ (set(all_nodes.keys()) - set(g_nodes) - set(master_nodes)))
+
+ # Find a schedulable nodes in other nodes if not skip the test
+ initiator_nodes = [
+ node for node in o_nodes if all_nodes[node] == '<none>']
+
+ if not initiator_nodes:
+ self.skipTest('Sufficient schedulable nodes are not available')
+
+ # Get schedulable node
+ schedulable_nodes = oc_get_schedulable_nodes(self.node)
+
+ # Get the list of nodes which needs to be marked as unschedulable
+ unschedule_nodes = list(
+ set(schedulable_nodes) - set([initiator_nodes[0]]))
+
+ # Mark all schedulable, gluster and master nodes as unschedulable
+ # except one node on which app pods wil run
+ oc_adm_manage_node(
+ self.node, '--schedulable=false', nodes=unschedule_nodes)
+ self.addCleanup(
+ oc_adm_manage_node, self.node, '--schedulable=true',
+ nodes=unschedule_nodes)
+
+ return initiator_nodes[0]
+
+ def test_initiator_and_target_on_diff_node_abrupt_reboot_of_initiator_node(
+ self):
+ """Abrupt reboot initiator node to make sure paths rediscovery is
+ happening.
+ """
+ ini_node = self.get_initiator_node_and_mark_other_nodes_unschedulable()
+
+ # Create 5 PVC's and 5 DC's
+ pvcs = self.create_and_wait_for_pvcs(pvc_amount=5)
+ dcs = self.create_dcs_with_pvc(pvcs)
+
+ for pvc, (dc_name, _) in dcs.items():
+ self.verify_iscsi_sessions_and_multipath(pvc, dc_name)
+
+ # Run I/O on app pods
+ _file, base_size, count = '/mnt/file', 4096, 1000
+ file_size = base_size * count
+ cmd_run_io = 'dd if=/dev/urandom of=%s bs=%s count=%s' % (
+ _file, base_size, count)
+
+ for _, pod_name in dcs.values():
+ oc_rsh(self.node, pod_name, cmd_run_io)
+
+ # Reboot initiator node where all the app pods are running
+ node_reboot_by_command(ini_node)
+ wait_for_ocp_node_be_ready(self.node, ini_node)
+
+ # Wait for pods to be ready after node reboot
+ pod_names = scale_dcs_pod_amount_and_wait(
+ self.node, [dc[0] for dc in dcs.values()])
+
+ for pvc, (dc_name, _) in dcs.items():
+ pod_name = pod_names[dc_name][0]
+ # Verify file
+ _, out, _ = oc_rsh(self.node, pod_name, 'ls -l %s' % _file)
+ msg = ("Expected size %s of file '%s' is not present "
+ "in out '%s'" % (file_size, _file, out))
+ self.assertIn(six.text_type(file_size), out, msg)
+
+ self.verify_iscsi_sessions_and_multipath(pvc, dc_name)