diff options
Diffstat (limited to 'openshift-storage-libs/openshiftstoragelibs')
3 files changed, 140 insertions, 5 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py index ebf5c77f..4ca48f0d 100644 --- a/openshift-storage-libs/openshiftstoragelibs/baseclass.py +++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py @@ -7,6 +7,7 @@ import six from openshiftstoragelibs import command from openshiftstoragelibs.exceptions import ( + CloudProviderError, ConfigError, ExecutionError, ) @@ -26,6 +27,8 @@ from openshiftstoragelibs.heketi_ops import ( from openshiftstoragelibs.node_ops import ( node_add_iptables_rules, node_delete_iptables_rules, + power_off_vm_by_name, + power_on_vm_by_name, ) from openshiftstoragelibs.openshift_ops import ( get_block_provisioner, @@ -41,9 +44,12 @@ from openshiftstoragelibs.openshift_ops import ( oc_label, scale_dcs_pod_amount_and_wait, switch_oc_project, + wait_for_gluster_pod_be_ready_on_specific_node, + wait_for_ocp_node_be_ready, wait_for_pvcs_be_bound, wait_for_pods_be_ready, wait_for_resources_absence, + wait_for_service_status_on_gluster_pod_or_node, ) from openshiftstoragelibs.openshift_storage_libs import ( get_iscsi_block_devices_by_path, @@ -549,6 +555,64 @@ class BaseClass(unittest.TestCase): g.log.warn(msg) return super(BaseClass, cls).doClassCleanups() + def power_on_vm(self, vm_name): + try: + power_on_vm_by_name(vm_name) + except CloudProviderError as e: + # Try to power on VM, if it raises already powered On error + # then don't raise exception. + if 'VM %s is already powered On' % vm_name not in e: + raise + + def power_off_vm(self, vm_name): + self.addCleanup(self.power_on_vm, vm_name) + power_off_vm_by_name(vm_name) + + def power_on_gluster_node_vm( + self, vm_name, gluster_hostname, timeout=300, wait_step=3): + # NOTE(Nitin Goyal): Same timeout is used for all functions. + + # Bring up the target node + power_on_vm_by_name(vm_name) + + # Wait for gluster node and pod to be ready + if self.is_containerized_gluster(): + wait_for_ocp_node_be_ready( + self.node, gluster_hostname, + timeout=timeout, wait_step=wait_step) + wait_for_gluster_pod_be_ready_on_specific_node( + self.node, gluster_hostname, + timeout=timeout, wait_step=wait_step) + + # Wait for gluster services to be up + for service in ('glusterd', 'gluster-blockd'): + wait_for_service_status_on_gluster_pod_or_node( + self.node, service, 'active', 'running', gluster_hostname, + raise_on_error=False, timeout=timeout, wait_step=wait_step) + + def power_off_gluster_node_vm( + self, vm_name, gluster_hostname, timeout=300, wait_step=3): + # NOTE(Nitin Goyal): Same timeout is used for all functions. + + # Wait for gluster services to be up in cleanup + for service in ('gluster-blockd', 'glusterd'): + self.addCleanup( + wait_for_service_status_on_gluster_pod_or_node, + self.node, service, 'active', 'running', gluster_hostname, + raise_on_error=False, timeout=timeout, wait_step=wait_step) + + # Wait for gluster pod to be up and node to be ready in cleanup + if self.is_containerized_gluster(): + self.addCleanup( + wait_for_gluster_pod_be_ready_on_specific_node, self.node, + gluster_hostname, timeout=timeout, wait_step=wait_step) + self.addCleanup( + wait_for_ocp_node_be_ready, self.node, gluster_hostname, + timeout=timeout, wait_step=wait_step) + + # Power off vm + self.power_off_vm(vm_name) + class GlusterBlockBaseClass(BaseClass): """Base class for gluster-block test cases.""" diff --git a/openshift-storage-libs/openshiftstoragelibs/cloundproviders/vmware.py b/openshift-storage-libs/openshiftstoragelibs/cloundproviders/vmware.py index fe69b532..1d4d4c38 100644 --- a/openshift-storage-libs/openshiftstoragelibs/cloundproviders/vmware.py +++ b/openshift-storage-libs/openshiftstoragelibs/cloundproviders/vmware.py @@ -198,6 +198,8 @@ class VmWare(object): g.log.error(msg) raise exceptions.CloudProviderError(msg) + # TODO(Nitin Goyal): Need to raise exact same below exception in other + # cloud providers as well in future e.g. AWS etc. if vm[0].summary.runtime.powerState == 'poweredOn': msg = 'VM %s is already powered On' % vm_name g.log.error(msg) @@ -228,6 +230,8 @@ class VmWare(object): g.log.error(msg) raise exceptions.CloudProviderError(msg) + # TODO(Nitin Goyal): Need to raise exact same below exception in other + # cloud providers as well in future e.g. AWS etc. if vm[0].summary.runtime.powerState == 'poweredOff': msg = 'VM %s is already powered Off' % vm_name g.log.error(msg) diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py index a20e91e6..56b06297 100644 --- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py +++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py @@ -467,7 +467,8 @@ def oc_delete(ocp_node, rtype, name, raise_on_absence=True): g.log.info('Deleted resource: %r %r', rtype, name) -def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None): +def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None, + field_selector=None): """Get an OCP resource by custom column names. Args: @@ -477,6 +478,8 @@ def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None): name (str|None): Name of the resource to fetch. selector (str|list|None): Column Name or list of column names select to. + field_selector (str|list|None): object field selector + which looks like following: 'spec.nodeName=foo_node_1' Returns: list: List containting data about the resource custom column Raises: @@ -489,15 +492,25 @@ def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None): """ cmd = ['oc', 'get', rtype, '--no-headers'] - cmd.append('-o=custom-columns=%s' % ( - ','.join(custom) if isinstance(custom, list) else custom)) + if name: + cmd.append(name) if selector: cmd.append('--selector %s' % ( ','.join(selector) if isinstance(selector, list) else selector)) - if name: - cmd.append(name) + if field_selector: + # NOTE(Nitin Goyal): Add field-selector parameters to custom because it + # is not supported in ocp 3.6 and 3.7 and filter them via python later + custom = ','.join(custom) if isinstance(custom, list) else custom + field_selector = (field_selector.split(',') if isinstance( + field_selector, six.string_types) else field_selector) + + for fs in field_selector: + custom += ',:' + re.split('=|!=', fs)[0] + + cmd.append('-o=custom-columns=%s' % ( + ','.join(custom) if isinstance(custom, list) else custom)) out = command.cmd_run(cmd, hostname=ocp_node) @@ -508,7 +521,21 @@ def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None): for line in (out.strip()).split('\n'): out_list.append( list(filter(None, map(str.strip, line.split(' '))))) + + if not field_selector: return out_list + # Filter out field-selector parameters + for fs in field_selector[::-1]: + fs_value = re.split('=|!=', fs)[1] + found = fs.find('!=') + for out in out_list[:]: + # Not equalto in fs and value present in list then remove it + if (found > 0 and out[-1] == fs_value + # Equalto in fs and value does not match to fs then remove + or found == -1 and out[-1] != fs_value): + out_list.remove(out) + out.pop() + return out_list def get_block_provisioner(ocp_node): @@ -802,6 +829,46 @@ def get_gluster_pod_names_by_pvc_name( return None +def wait_for_gluster_pod_be_ready_on_specific_node( + ocp_client_node, gluster_hostname, selector='glusterfs=storage-pod', + timeout=300, wait_step=10): + """Wait for gluster pod to be ready on specific node. + + Args: + ocp_client_node (str): Node to execute OCP commands on. + gluster_hostname (str): Name of node hosting gluster pod. + selector (str): Selector for gluster pod. + + Returns: + None + """ + g_pod_name = get_gluster_pod_name_for_specific_node( + ocp_client_node, gluster_hostname, selector=selector) + wait_for_pod_be_ready( + ocp_client_node, g_pod_name, timeout=timeout, wait_step=wait_step) + + +def get_gluster_pod_name_for_specific_node( + ocp_client_node, gluster_hostname, selector='glusterfs=storage-pod'): + """Get gluster pod name on specific gluster node. + + Args: + ocp_client_node (str): Node to execute OCP commands on. + gluster_hostname (str): Name of node hosting gluster pod. + selector (str): Selector for gluster pod. + + Returns: + str: Name of the gluster pod + """ + g_pod_name = oc_get_custom_resource( + ocp_client_node, 'pod', ':.metadata.name', selector=selector, + field_selector='spec.nodeName=%s' % gluster_hostname) + if not g_pod_name: + raise AssertionError( + 'Gluster pod was not found for node %s' % gluster_hostname) + return g_pod_name[0][0] + + def cmd_run_on_gluster_pod_or_node( ocp_client_node, cmd, gluster_node=None, raise_on_error=True): """Run shell command on either Gluster PODs or Gluster nodes. |