Add TC restart initiator node when gluster node is down

Add new TC where we verify iscsi rediscovery is happening properly
on restart of initiator nodes when one gluster node is down.

Change-Id: I515bb27d43843f7c19c4a12b6531a212e9c3285a

1 files changed, 64 insertions, 0 deletions

diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
index ebf5c77f..4ca48f0d 100644
--- a/openshift-storage-libs/openshiftstoragelibs/baseclass.py
+++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
@@ -7,6 +7,7 @@ import six
 
 from openshiftstoragelibs import command
 from openshiftstoragelibs.exceptions import (
+    CloudProviderError,
     ConfigError,
     ExecutionError,
 )
@@ -26,6 +27,8 @@ from openshiftstoragelibs.heketi_ops import (
 from openshiftstoragelibs.node_ops import (
     node_add_iptables_rules,
     node_delete_iptables_rules,
+    power_off_vm_by_name,
+    power_on_vm_by_name,
 )
 from openshiftstoragelibs.openshift_ops import (
     get_block_provisioner,
@@ -41,9 +44,12 @@ from openshiftstoragelibs.openshift_ops import (
     oc_label,
     scale_dcs_pod_amount_and_wait,
     switch_oc_project,
+    wait_for_gluster_pod_be_ready_on_specific_node,
+    wait_for_ocp_node_be_ready,
     wait_for_pvcs_be_bound,
     wait_for_pods_be_ready,
     wait_for_resources_absence,
+    wait_for_service_status_on_gluster_pod_or_node,
 )
 from openshiftstoragelibs.openshift_storage_libs import (
     get_iscsi_block_devices_by_path,
@@ -549,6 +555,64 @@ class BaseClass(unittest.TestCase):
                 g.log.warn(msg)
         return super(BaseClass, cls).doClassCleanups()
 
+    def power_on_vm(self, vm_name):
+        try:
+            power_on_vm_by_name(vm_name)
+        except CloudProviderError as e:
+            # Try to power on VM, if it raises already powered On error
+            # then don't raise exception.
+            if 'VM %s is already powered On' % vm_name not in e:
+                raise
+
+    def power_off_vm(self, vm_name):
+        self.addCleanup(self.power_on_vm, vm_name)
+        power_off_vm_by_name(vm_name)
+
+    def power_on_gluster_node_vm(
+            self, vm_name, gluster_hostname, timeout=300, wait_step=3):
+        # NOTE(Nitin Goyal): Same timeout is used for all functions.
+
+        # Bring up the target node
+        power_on_vm_by_name(vm_name)
+
+        # Wait for gluster node and pod to be ready
+        if self.is_containerized_gluster():
+            wait_for_ocp_node_be_ready(
+                self.node, gluster_hostname,
+                timeout=timeout, wait_step=wait_step)
+            wait_for_gluster_pod_be_ready_on_specific_node(
+                self.node, gluster_hostname,
+                timeout=timeout, wait_step=wait_step)
+
+        # Wait for gluster services to be up
+        for service in ('glusterd', 'gluster-blockd'):
+            wait_for_service_status_on_gluster_pod_or_node(
+                self.node, service, 'active', 'running', gluster_hostname,
+                raise_on_error=False, timeout=timeout, wait_step=wait_step)
+
+    def power_off_gluster_node_vm(
+            self, vm_name, gluster_hostname, timeout=300, wait_step=3):
+        # NOTE(Nitin Goyal): Same timeout is used for all functions.
+
+        # Wait for gluster services to be up in cleanup
+        for service in ('gluster-blockd', 'glusterd'):
+            self.addCleanup(
+                wait_for_service_status_on_gluster_pod_or_node,
+                self.node, service, 'active', 'running', gluster_hostname,
+                raise_on_error=False, timeout=timeout, wait_step=wait_step)
+
+        # Wait for gluster pod to be up and node to be ready in cleanup
+        if self.is_containerized_gluster():
+            self.addCleanup(
+                wait_for_gluster_pod_be_ready_on_specific_node, self.node,
+                gluster_hostname, timeout=timeout, wait_step=wait_step)
+            self.addCleanup(
+                wait_for_ocp_node_be_ready, self.node, gluster_hostname,
+                timeout=timeout, wait_step=wait_step)
+
+        # Power off vm
+        self.power_off_vm(vm_name)
+
 
 class GlusterBlockBaseClass(BaseClass):
     """Base class for gluster-block test cases."""