"""Library for openshift operations. Various utility functions for interacting with OCP/OpenShift. """ import base64 import json import re import types from glusto.core import Glusto as g from glustolibs.gluster import volume_ops from glustolibs.gluster.brick_libs import ( are_bricks_online, get_all_bricks, get_online_bricks_list) import mock import yaml from cnslibs.common import command from cnslibs.common import exceptions from cnslibs.common import podcmd from cnslibs.common import utils from cnslibs.common import waiter from cnslibs.common.heketi_ops import ( heketi_blockvolume_info, heketi_volume_info, ) PODS_WIDE_RE = re.compile( '(\S+)\s+(\S+)\s+(\w+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+).*\n') def oc_get_pods(ocp_node): """Gets the pods info with 'wide' option in the current project. Args: ocp_node (str): Node in which ocp command will be executed. Returns: dict : dict of pods info in the current project. """ cmd = "oc get -o wide --no-headers=true pods" ret, out, err = g.run(ocp_node, cmd) if ret != 0: g.log.error("Failed to get ocp pods on node %s" % ocp_node) raise AssertionError('failed to get pods: %r' % (err,)) return _parse_wide_pods_output(out) def _parse_wide_pods_output(output): """Parse the output of `oc get -o wide pods`. """ # Interestingly, the output of get pods is "cooked" in such a way that # the values in the ready, status, & restart fields are not accessible # from YAML/JSON/templating forcing us to scrape the output for # these values # (at the time of this writing the logic is in # printPodBase in kubernetes/pkg/printers/internalversion/printers.go ) # Possibly obvious, but if you don't need those values you can # use the YAML output directly. # # TODO: Add unit tests for this parser pods_info = {} for each_pod_info in PODS_WIDE_RE.findall(output): pods_info[each_pod_info[0]] = { 'ready': each_pod_info[1], 'status': each_pod_info[2], 'restarts': each_pod_info[3], 'age': each_pod_info[4], 'ip': each_pod_info[5], 'node': each_pod_info[6], } return pods_info def oc_get_pods_full(ocp_node): """Gets all the pod info via YAML in the current project. Args: ocp_node (str): Node in which ocp command will be executed. Returns: dict: The YAML output converted to python objects (a top-level dict) """ cmd = "oc get -o yaml pods" ret, out, err = g.run(ocp_node, cmd) if ret != 0: g.log.error("Failed to get ocp pods on node %s" % ocp_node) raise AssertionError('failed to get pods: %r' % (err,)) return yaml.load(out) def get_ocp_gluster_pod_names(ocp_node): """Gets the gluster pod names in the current project. Args: ocp_node (str): Node in which ocp command will be executed. Returns: list : list of gluster pod names in the current project. Empty list, if there are no gluster pods. Example: get_ocp_gluster_pod_names(ocp_node) """ pod_names = oc_get_pods(ocp_node).keys() return [pod for pod in pod_names if pod.startswith('glusterfs-')] def oc_login(ocp_node, username, password): """Login to ocp master node. Args: ocp_node (str): Node in which ocp command will be executed. username (str): username of ocp master node to login. password (str): password of ocp master node to login. Returns: bool : True on successful login to ocp master node. False otherwise Example: oc_login(ocp_node, "test","test") """ cmd = "oc login --username=%s --password=%s" % (username, password) ret, _, _ = g.run(ocp_node, cmd) if ret != 0: g.log.error("Failed to login to ocp master node %s" % ocp_node) return False return True def switch_oc_project(ocp_node, project_name): """Switch to the given project. Args: ocp_node (str): Node in which ocp command will be executed. project_name (str): Project name. Returns: bool : True on switching to given project. False otherwise Example: switch_oc_project(ocp_node, "storage-project") """ cmd = "oc project %s" % project_name ret, _, _ = g.run(ocp_node, cmd) if ret != 0: g.log.error("Failed to switch to project %s" % project_name) return False return True def oc_rsync(ocp_node, pod_name, src_dir_path, dest_dir_path): """Sync file from 'src_dir_path' path on ocp_node to 'dest_dir_path' path on 'pod_name' using 'oc rsync' command. Args: ocp_node (str): Node on which oc rsync command will be executed pod_name (str): Name of the pod on which source directory to be mounted src_dir_path (path): Source path from which directory to be mounted dest_dir_path (path): destination path to which directory to be mounted """ ret, out, err = g.run(ocp_node, ['oc', 'rsync', src_dir_path, '%s:%s' % (pod_name, dest_dir_path)]) if ret != 0: error_msg = 'failed to sync directory in pod: %r; %r' % (out, err) g.log.error(error_msg) raise AssertionError(error_msg) def oc_rsh(ocp_node, pod_name, command, log_level=None): """Run a command in the ocp pod using `oc rsh`. Args: ocp_node (str): Node on which oc rsh command will be executed. pod_name (str): Name of the pod on which the command will be executed. command (str|list): command to run. log_level (str|None): log level to be passed to glusto's run method. Returns: A tuple consisting of the command return code, stdout, and stderr. """ prefix = ['oc', 'rsh', pod_name] if isinstance(command, types.StringTypes): cmd = ' '.join(prefix + [command]) else: cmd = prefix + command # unpack the tuple to make sure our return value exactly matches # our docstring ret, stdout, stderr = g.run(ocp_node, cmd, log_level=log_level) return (ret, stdout, stderr) def oc_create(ocp_node, value, value_type='file'): """Create a resource based on the contents of the given file name. Args: ocp_node (str): Node on which the ocp command will run value (str): Filename (on remote) or file data to be passed to oc create command. value_type (str): either 'file' or 'stdin'. Raises: AssertionError: Raised when resource fails to create. """ if value_type == 'file': cmd = ['oc', 'create', '-f', value] else: cmd = ['echo', '\'%s\'' % value, '|', 'oc', 'create', '-f', '-'] ret, out, err = g.run(ocp_node, cmd) if ret != 0: msg = 'Failed to create resource: %r; %r' % (out, err) g.log.error(msg) raise AssertionError(msg) g.log.info('Created resource from %s.' % value_type) def oc_process(ocp_node, params, filename): """Create a resource template based on the contents of the given filename and params provided. Args: ocp_node (str): Node on which the ocp command will run filename (str): Filename (on remote) to be passed to oc process command. Returns: template generated through process command Raises: AssertionError: Raised when resource fails to create. """ ret, out, err = g.run(ocp_node, ['oc', 'process', '-f', filename, params]) if ret != 0: error_msg = 'failed to create process: %r; %r' % (out, err) g.log.error(error_msg) raise AssertionError(error_msg) g.log.info('Created resource from file (%s)', filename) return out def oc_create_secret(hostname, secret_name_prefix="autotests-secret-", namespace="default", data_key="password", secret_type="kubernetes.io/glusterfs"): """Create secret using data provided as stdin input. Args: hostname (str): Node on which 'oc create' command will be executed. secret_name_prefix (str): secret name will consist of this prefix and random str. namespace (str): name of a namespace to create a secret in data_key (str): plain text value for secret which will be transformed into base64 string automatically. secret_type (str): type of the secret, which will be created. Returns: name of a secret """ secret_name = "%s-%s" % (secret_name_prefix, utils.get_random_str()) secret_data = json.dumps({ "apiVersion": "v1", "data": {"key": base64.b64encode(data_key)}, "kind": "Secret", "metadata": { "name": secret_name, "namespace": namespace, }, "type": secret_type, }) oc_create(hostname, secret_data, 'stdin') return secret_name def oc_create_sc(hostname, sc_name_prefix="autotests-sc", provisioner="kubernetes.io/glusterfs", allow_volume_expansion=False, **parameters): """Create storage class using data provided as stdin input. Args: hostname (str): Node on which 'oc create' command will be executed. sc_name_prefix (str): sc name will consist of this prefix and random str. provisioner (str): name of the provisioner allow_volume_expansion (bool): Set it to True if need to allow volume expansion. Kvargs: All the keyword arguments are expected to be key and values of 'parameters' section for storage class. """ allowed_parameters = ( 'resturl', 'secretnamespace', 'restuser', 'secretname', 'restauthenabled', 'restsecretnamespace', 'restsecretname', 'hacount', 'clusterids', 'chapauthenabled', 'volumenameprefix', 'volumeoptions', 'volumetype' ) for parameter in parameters.keys(): if parameter.lower() not in allowed_parameters: parameters.pop(parameter) sc_name = "%s-%s" % (sc_name_prefix, utils.get_random_str()) sc_data = json.dumps({ "kind": "StorageClass", "apiVersion": "storage.k8s.io/v1", "metadata": {"name": sc_name}, "provisioner": provisioner, "parameters": parameters, "allowVolumeExpansion": allow_volume_expansion, }) oc_create(hostname, sc_data, 'stdin') return sc_name def oc_create_pvc(hostname, sc_name, pvc_name_prefix="autotests-pvc", pvc_size=1): """Create PVC using data provided as stdin input. Args: hostname (str): Node on which 'oc create' command will be executed. sc_name (str): name of a storage class to create PVC in. pvc_name_prefix (str): PVC name will consist of this prefix and random str. pvc_size (int/str): size of PVC in Gb """ pvc_name = "%s-%s" % (pvc_name_prefix, utils.get_random_str()) pvc_data = json.dumps({ "kind": "PersistentVolumeClaim", "apiVersion": "v1", "metadata": { "name": pvc_name, "annotations": { "volume.beta.kubernetes.io/storage-class": sc_name, }, }, "spec": { "accessModes": ["ReadWriteOnce"], "resources": {"requests": {"storage": "%sGi" % pvc_size}} }, }) oc_create(hostname, pvc_data, 'stdin') return pvc_name def oc_create_app_dc_with_io( hostname, pvc_name, dc_name_prefix="autotests-dc-with-app-io", replicas=1, space_to_use=1048576): """Create DC with app PODs and attached PVC, constantly running I/O. Args: hostname (str): Node on which 'oc create' command will be executed. pvc_name (str): name of the Persistent Volume Claim to attach to the application PODs where constant I\O will run. dc_name_prefix (str): DC name will consist of this prefix and random str. replicas (int): amount of application POD replicas. space_to_use (int): value in bytes which will be used for I\O. """ dc_name = "%s-%s" % (dc_name_prefix, utils.get_random_str()) container_data = { "name": dc_name, "image": "cirros", "volumeMounts": [{"mountPath": "/mnt", "name": dc_name}], "command": ["sh"], "args": [ "-ec", "trap \"rm -f /mnt/random-data-$HOSTNAME.log ; exit 0\" SIGTERM; " "while true; do " " (mount | grep '/mnt') && " " (head -c %s < /dev/urandom > /mnt/random-data-$HOSTNAME.log) ||" " exit 1; " " sleep 1 ; " "done" % space_to_use, ], "livenessProbe": { "initialDelaySeconds": 3, "periodSeconds": 3, "exec": {"command": [ "sh", "-ec", "mount | grep '/mnt' && " " head -c 1 < /dev/urandom >> /mnt/random-data-$HOSTNAME.log" ]}, }, } dc_data = json.dumps({ "kind": "DeploymentConfig", "apiVersion": "v1", "metadata": {"name": dc_name}, "spec": { "replicas": replicas, "triggers": [{"type": "ConfigChange"}], "paused": False, "revisionHistoryLimit": 2, "template": { "metadata": {"labels": {"name": dc_name}}, "spec": { "restartPolicy": "Always", "volumes": [{ "name": dc_name, "persistentVolumeClaim": {"claimName": pvc_name}, }], "containers": [container_data], "terminationGracePeriodSeconds": 20, } } } }) oc_create(hostname, dc_data, 'stdin') return dc_name def oc_create_tiny_pod_with_volume(hostname, pvc_name, pod_name_prefix='', mount_path='/mnt'): """Create tiny POD from image in 10Mb with attached volume at /mnt""" pod_name = "%s-%s" % (pod_name_prefix, utils.get_random_str()) pod_data = json.dumps({ "apiVersion": "v1", "kind": "Pod", "metadata": { "name": pod_name, }, "spec": { "terminationGracePeriodSeconds": 20, "containers": [{ "name": pod_name, "image": "cirros", # noqa: 10 Mb! linux image "volumeMounts": [{"mountPath": mount_path, "name": "vol"}], "command": [ "/bin/sh", "-ec", "trap 'exit 0' SIGTERM ; " "while :; do echo '.'; sleep 5 ; done", ] }], "volumes": [{ "name": "vol", "persistentVolumeClaim": {"claimName": pvc_name}, }], "restartPolicy": "Never", } }) oc_create(hostname, pod_data, 'stdin') return pod_name def oc_delete(ocp_node, rtype, name, raise_on_absence=True): """Delete an OCP resource by name. Args: ocp_node (str): Node on which the ocp command will run. rtype (str): Name of the resource type (pod, storageClass, etc). name (str): Name of the resource to delete. raise_on_absence (bool): if resource absent raise exception if value is true, else return default value: True """ if not oc_get_yaml(ocp_node, rtype, name, raise_on_error=raise_on_absence): return ret, out, err = g.run(ocp_node, ['oc', 'delete', rtype, name]) if ret != 0: g.log.error('Failed to delete resource: %s, %s: %r; %r', rtype, name, out, err) raise AssertionError('failed to delete resource: %r; %r' % (out, err)) g.log.info('Deleted resource: %r %r', rtype, name) def oc_get_yaml(ocp_node, rtype, name=None, raise_on_error=True): """Get an OCP resource by name. Args: ocp_node (str): Node on which the ocp command will run. rtype (str): Name of the resource type (pod, storageClass, etc). name (str|None): Name of the resource to fetch. raise_on_error (bool): If set to true a failure to fetch resource inforation will raise an error, otherwise an empty dict will be returned. Returns: dict: Dictionary containting data about the resource Raises: AssertionError: Raised when unable to get resource and `raise_on_error` is true. """ cmd = ['oc', 'get', '-oyaml', rtype] if name is not None: cmd.append(name) ret, out, err = g.run(ocp_node, cmd) if ret != 0: g.log.error('Failed to get %s: %s: %r', rtype, name, err) if raise_on_error: raise AssertionError('failed to get %s: %s: %r' % (rtype, name, err)) return {} return yaml.load(out) def oc_get_pvc(ocp_node, name): """Get information on a persistant volume claim. Args: ocp_node (str): Node on which the ocp command will run. name (str): Name of the PVC. Returns: dict: Dictionary containting data about the PVC. """ return oc_get_yaml(ocp_node, 'pvc', name) def oc_get_pv(ocp_node, name): """Get information on a persistant volume. Args: ocp_node (str): Node on which the ocp command will run. name (str): Name of the PV. Returns: dict: Dictionary containting data about the PV. """ return oc_get_yaml(ocp_node, 'pv', name) def oc_get_all_pvs(ocp_node): """Get information on all persistent volumes. Args: ocp_node (str): Node on which the ocp command will run. Returns: dict: Dictionary containting data about the PV. """ return oc_get_yaml(ocp_node, 'pv', None) def create_namespace(hostname, namespace): ''' This function creates namespace Args: hostname (str): hostname on which we need to create namespace namespace (str): project name Returns: bool: True if successful and if already exists, otherwise False ''' cmd = "oc new-project %s" % namespace ret, out, err = g.run(hostname, cmd, "root") if ret == 0: g.log.info("new namespace %s successfully created" % namespace) return True output = out.strip().split("\n")[0] if "already exists" in output: g.log.info("namespace %s already exists" % namespace) return True g.log.error("failed to create namespace %s" % namespace) return False def wait_for_resource_absence(ocp_node, rtype, name, interval=10, timeout=120): _waiter = waiter.Waiter(timeout=timeout, interval=interval) for w in _waiter: try: oc_get_yaml(ocp_node, rtype, name, raise_on_error=True) except AssertionError: break if rtype == 'pvc': cmd = "oc get pv -o=custom-columns=:.spec.claimRef.name | grep %s" % ( name) for w in _waiter: ret, out, err = g.run(ocp_node, cmd, "root") if ret != 0: break if w.expired: error_msg = "%s '%s' still exists after waiting for it %d seconds" % ( rtype, name, timeout) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) def scale_dc_pod_amount_and_wait(hostname, dc_name, pod_amount=1, namespace=None): ''' This function scales pod and waits If pod_amount 0 waits for its absence If pod_amount => 1 waits for all pods to be ready Args: hostname (str): Node on which the ocp command will run dc_name (str): Name of heketi dc namespace (str): Namespace pod_amount (int): Number of heketi pods to scale ex: 0, 1 or 2 default 1 ''' namespace_arg = "--namespace=%s" % namespace if namespace else "" heketi_scale_cmd = "oc scale --replicas=%d dc/%s %s" % ( pod_amount, dc_name, namespace_arg) ret, out, err = g.run(hostname, heketi_scale_cmd, "root") if ret != 0: error_msg = ("failed to execute cmd %s " "out- %s err %s" % (heketi_scale_cmd, out, err)) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) get_heketi_podname_cmd = ( "oc get pods --all-namespaces -o=custom-columns=:.metadata.name " "--no-headers=true " "--selector deploymentconfig=%s" % dc_name) ret, out, err = g.run(hostname, get_heketi_podname_cmd) if ret != 0: error_msg = ("failed to execute cmd %s " "out- %s err %s" % (get_heketi_podname_cmd, out, err)) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) pod_list = out.strip().split("\n") for pod in pod_list: if pod_amount == 0: wait_for_resource_absence(hostname, 'pod', pod) else: wait_for_pod_be_ready(hostname, pod) def get_gluster_pod_names_by_pvc_name(ocp_node, pvc_name): """Get Gluster POD names, whose nodes store bricks for specified PVC. Args: ocp_node (str): Node to execute OCP commands on. pvc_name (str): Name of a PVC to get related Gluster PODs. Returns: list: List of dicts, which consist of following 3 key-value pairs: pod_name=, host_name=, host_ip= """ # Check storage provisioner sp_cmd = ( 'oc get pvc %s --no-headers -o=custom-columns=' ':.metadata.annotations."volume\.beta\.kubernetes\.io\/' 'storage\-provisioner"' % pvc_name) sp_raw = command.cmd_run(sp_cmd, hostname=ocp_node) sp = sp_raw.strip() # Get node IPs if sp == "kubernetes.io/glusterfs": pv_info = get_gluster_vol_info_by_pvc_name(ocp_node, pvc_name) gluster_pod_nodes_ips = [ brick["name"].split(":")[0] for brick in pv_info["bricks"]["brick"] ] elif sp == "gluster.org/glusterblock": get_gluster_pod_node_ip_cmd = ( r"""oc get pv --template '{{range .items}}""" + r"""{{if eq .spec.claimRef.name "%s"}}""" + r"""{{.spec.iscsi.targetPortal}}{{" "}}""" + r"""{{.spec.iscsi.portals}}{{end}}{{end}}'""") % ( pvc_name) node_ips_raw = command.cmd_run( get_gluster_pod_node_ip_cmd, hostname=ocp_node) node_ips_raw = node_ips_raw.replace( "[", " ").replace("]", " ").replace(",", " ") gluster_pod_nodes_ips = [ s.strip() for s in node_ips_raw.split(" ") if s.strip() ] else: assert False, "Unexpected storage provisioner: %s" % sp # Get node names get_node_names_cmd = ( "oc get node -o wide | grep -e '%s ' | awk '{print $1}'" % ( " ' -e '".join(gluster_pod_nodes_ips))) gluster_pod_node_names = command.cmd_run( get_node_names_cmd, hostname=ocp_node) gluster_pod_node_names = [ node_name.strip() for node_name in gluster_pod_node_names.split("\n") if node_name.strip() ] node_count = len(gluster_pod_node_names) err_msg = "Expected more than one node hosting Gluster PODs. Got '%s'." % ( node_count) assert (node_count > 1), err_msg # Get Gluster POD names which are located on the filtered nodes get_pod_name_cmd = ( "oc get pods --all-namespaces " "-o=custom-columns=:.metadata.name,:.spec.nodeName,:.status.hostIP | " "grep 'glusterfs-' | grep -e '%s '" % "' -e '".join( gluster_pod_node_names) ) out = command.cmd_run( get_pod_name_cmd, hostname=ocp_node) data = [] for line in out.split("\n"): pod_name, host_name, host_ip = [ el.strip() for el in line.split(" ") if el.strip()] data.append({ "pod_name": pod_name, "host_name": host_name, "host_ip": host_ip, }) pod_count = len(data) err_msg = "Expected 3 or more Gluster PODs to be found. Actual is '%s'" % ( pod_count) assert (pod_count > 2), err_msg return data def get_gluster_vol_info_by_pvc_name(ocp_node, pvc_name): """Get Gluster volume info based on the PVC name. Args: ocp_node (str): Node to execute OCP commands on. pvc_name (str): Name of a PVC to get bound Gluster volume info. Returns: dict: Dictionary containting data about a Gluster volume. """ # Get PV ID from PVC get_pvc_cmd = "oc get pvc %s -o=custom-columns=:.spec.volumeName" % ( pvc_name) pv_name = command.cmd_run(get_pvc_cmd, hostname=ocp_node) assert pv_name, "PV name should not be empty: '%s'" % pv_name # Get volume ID from PV get_pv_cmd = "oc get pv %s -o=custom-columns=:.spec.glusterfs.path" % ( pv_name) vol_id = command.cmd_run(get_pv_cmd, hostname=ocp_node) assert vol_id, "Gluster volume ID should not be empty: '%s'" % vol_id # Get name of one the Gluster PODs gluster_pod = get_ocp_gluster_pod_names(ocp_node)[1] # Get Gluster volume info vol_info_cmd = "oc exec %s -- gluster v info %s --xml" % ( gluster_pod, vol_id) vol_info = command.cmd_run(vol_info_cmd, hostname=ocp_node) # Parse XML output to python dict with mock.patch('glusto.core.Glusto.run', return_value=(0, vol_info, '')): vol_info = volume_ops.get_volume_info(vol_id) vol_info = vol_info[list(vol_info.keys())[0]] vol_info["gluster_vol_id"] = vol_id return vol_info def get_gluster_blockvol_info_by_pvc_name(ocp_node, heketi_server_url, pvc_name): """Get Gluster block volume info based on the PVC name. Args: ocp_node (str): Node to execute OCP commands on. heketi_server_url (str): Heketi server url pvc_name (str): Name of a PVC to get bound Gluster block volume info. Returns: dict: Dictionary containting data about a Gluster block volume. """ # Get block volume Name and ID from PV which is bound to our PVC get_block_vol_data_cmd = ( 'oc get pv --no-headers -o custom-columns=' ':.metadata.annotations.glusterBlockShare,' ':.metadata.annotations."gluster\.org\/volume\-id",' ':.spec.claimRef.name | grep "%s"' % pvc_name) out = command.cmd_run(get_block_vol_data_cmd, hostname=ocp_node) parsed_out = filter(None, map(str.strip, out.split(" "))) assert len(parsed_out) == 3, "Expected 3 fields in following: %s" % out block_vol_name, block_vol_id = parsed_out[:2] # Get block hosting volume ID block_hosting_vol_id = heketi_blockvolume_info( ocp_node, heketi_server_url, block_vol_id, json=True )["blockhostingvolume"] # Get block hosting volume name by it's ID block_hosting_vol_name = heketi_volume_info( ocp_node, heketi_server_url, block_hosting_vol_id, json=True)['name'] # Get Gluster block volume info vol_info_cmd = "oc exec %s -- gluster-block info %s/%s --json" % ( get_ocp_gluster_pod_names(ocp_node)[0], block_hosting_vol_name, block_vol_name) return json.loads(command.cmd_run(vol_info_cmd, hostname=ocp_node)) def wait_for_pod_be_ready(hostname, pod_name, timeout=1200, wait_step=60): ''' This funciton waits for pod to be in ready state Args: hostname (str): hostname on which we want to check the pod status pod_name (str): pod_name for which we need the status timeout (int): timeout value,, default value is 1200 sec wait_step( int): wait step, default value is 60 sec Returns: bool: True if pod status is Running and ready state, otherwise Raise Exception ''' for w in waiter.Waiter(timeout, wait_step): # command to find pod status and its phase cmd = ("oc get pods %s -o=custom-columns=" ":.status.containerStatuses[0].ready," ":.status.phase") % pod_name ret, out, err = g.run(hostname, cmd, "root") if ret != 0: msg = ("failed to execute cmd %s" % cmd) g.log.error(msg) raise exceptions.ExecutionError(msg) output = out.strip().split() # command to find if pod is ready if output[0] == "true" and output[1] == "Running": g.log.info("pod %s is in ready state and is " "Running" % pod_name) return True elif output[1] == "Error": msg = ("pod %s status error" % pod_name) g.log.error(msg) raise exceptions.ExecutionError(msg) else: g.log.info("pod %s ready state is %s," " phase is %s," " sleeping for %s sec" % ( pod_name, output[0], output[1], wait_step)) continue if w.expired: err_msg = ("exceeded timeout %s for waiting for pod %s " "to be in ready state" % (timeout, pod_name)) g.log.error(err_msg) raise exceptions.ExecutionError(err_msg) def get_pod_name_from_dc(hostname, dc_name, timeout=1200, wait_step=60): ''' This funciton return pod_name from dc_name Args: hostname (str): hostname on which we can execute oc commands dc_name (str): deployment_confidg name timeout (int): timeout value default value is 1200 sec wait_step( int): wait step, default value is 60 sec Returns: str: pod_name if successful otherwise Raise Exception ''' cmd = ("oc get pods --all-namespaces -o=custom-columns=" ":.metadata.name " "--no-headers=true " "--selector deploymentconfig=%s" % dc_name) for w in waiter.Waiter(timeout, wait_step): ret, out, err = g.run(hostname, cmd, "root") if ret != 0: msg = ("failed to execute cmd %s" % cmd) g.log.error(msg) raise exceptions.ExecutionError(msg) output = out.strip() if output == "": g.log.info("podname for dc %s not found sleeping for " "%s sec" % (dc_name, wait_step)) continue else: g.log.info("podname is %s for dc %s" % ( output, dc_name)) return output if w.expired: err_msg = ("exceeded timeout %s for waiting for pod_name" "for dc %s " % (timeout, dc_name)) g.log.error(err_msg) raise exceptions.ExecutionError(err_msg) def get_pvc_status(hostname, pvc_name): ''' This function verifies the if pod is running Args: hostname (str): hostname on which we want to check the pvc status pvc_name (str): pod_name for which we need the status Returns: bool, status (str): True, status of pvc otherwise False, error message. ''' cmd = "oc get pvc | grep %s | awk '{print $2}'" % pvc_name ret, out, err = g.run(hostname, cmd, "root") if ret != 0: g.log.error("failed to execute cmd %s" % cmd) return False, err output = out.strip().split("\n")[0].strip() return True, output def verify_pvc_status_is_bound(hostname, pvc_name, timeout=120, wait_step=3): """Verify that PVC gets 'Bound' status in required time. Args: hostname (str): hostname on which we will execute oc commands pvc_name (str): name of PVC to check status of timeout (int): total time in seconds we are ok to wait for 'Bound' status of a PVC wait_step (int): time in seconds we will sleep before checking a PVC status again. Returns: None Raises: exceptions.ExecutionError in case of errors. """ pvc_not_found_counter = 0 for w in waiter.Waiter(timeout, wait_step): ret, output = get_pvc_status(hostname, pvc_name) if ret is not True: msg = ("Failed to execute 'get' command for '%s' PVC. " "Got following responce: %s" % (pvc_name, output)) g.log.error(msg) raise exceptions.ExecutionError(msg) if output == "": g.log.info("PVC '%s' not found, sleeping for %s " "sec." % (pvc_name, wait_step)) if pvc_not_found_counter > 0: msg = ("PVC '%s' has not been found 2 times already. " "Make sure you provided correct PVC name." % pvc_name) else: pvc_not_found_counter += 1 continue elif output == "Pending": g.log.info("PVC '%s' is in Pending state, sleeping for %s " "sec" % (pvc_name, wait_step)) continue elif output == "Bound": g.log.info("PVC '%s' is in Bound state." % pvc_name) return pvc_name elif output == "Error": msg = "PVC '%s' is in 'Error' state." % pvc_name g.log.error(msg) else: msg = "PVC %s has different status - %s" % (pvc_name, output) g.log.error(msg) if msg: raise AssertionError(msg) if w.expired: # for debug purpose to see why provisioning failed cmd = "oc describe pvc %s | grep ProvisioningFailed" % pvc_name ret, out, err = g.run(hostname, cmd, "root") g.log.info("cmd %s - out- %s err- %s" % (cmd, out, err)) msg = ("Exceeded timeout of '%s' seconds for verifying PVC '%s' " "to reach the 'Bound' status." % (timeout, pvc_name)) g.log.error(msg) raise AssertionError(msg) def oc_version(hostname): ''' Get Openshift version from oc version command Args: hostname (str): Node on which the ocp command will run. Returns: str : oc version if successful, otherwise raise Exception ''' cmd = "oc version | grep openshift | cut -d ' ' -f 2" ret, out, err = g.run(hostname, cmd, "root") if ret != 0: msg = ("failed to get oc version err %s; out %s" % (err, out)) g.log.error(msg) raise AssertionError(msg) if not out: error_msg = "Empty string found for oc version" g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) return out.strip() def resize_pvc(hostname, pvc_name, size): ''' Resize PVC Args: hostname (str): hostname on which we want to edit the pvc status pvc_name (str): pod_name for which we edit the storage capacity size (int): size of pvc to change Returns: bool: True, if successful otherwise raise Exception ''' cmd = ("oc patch pvc %s " "-p='{\"spec\": {\"resources\": {\"requests\": " "{\"storage\": \"%dGi\"}}}}'" % (pvc_name, size)) ret, out, err = g.run(hostname, cmd, "root") if ret != 0: error_msg = ("failed to execute cmd %s " "out- %s err %s" % (cmd, out, err)) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) g.log.info("successfully edited storage capacity" "of pvc %s . out- %s" % (pvc_name, out)) return True def verify_pvc_size(hostname, pvc_name, size, timeout=120, wait_step=5): ''' Verify size of PVC Args: hostname (str): hostname on which we want to verify the size of pvc pvc_name (str): pvc_name for which we verify its size size (int): size of pvc timeout (int): timeout value, verifies the size after wait_step value till timeout default value is 120 sec wait_step( int): wait step, default value is 5 sec Returns: bool: True, if successful otherwise raise Exception ''' cmd = ("oc get pvc %s -o=custom-columns=" ":.spec.resources.requests.storage," ":.status.capacity.storage" % pvc_name) for w in waiter.Waiter(timeout, wait_step): sizes = command.cmd_run(cmd, hostname=hostname).split() spec_size = int(sizes[0].replace("Gi", "")) actual_size = int(sizes[1].replace("Gi", "")) if spec_size == actual_size == size: g.log.info("verification of pvc %s of size %d " "successful" % (pvc_name, size)) return True else: g.log.info("sleeping for %s sec" % wait_step) continue err_msg = ("verification of pvc %s size of %d failed -" "spec_size- %d actual_size %d" % ( pvc_name, size, spec_size, actual_size)) g.log.error(err_msg) raise AssertionError(err_msg) def verify_pv_size(hostname, pv_name, size, timeout=120, wait_step=5): ''' Verify size of PV Args: hostname (str): hostname on which we want to verify the size of pv pv_name (str): pv_name for which we verify its size size (int): size of pv timeout (int): timeout value, verifies the size after wait_step value till timeout default value is 120 sec wait_step( int): wait step, default value is 5 sec Returns: bool: True, if successful otherwise raise Exception ''' cmd = ("oc get pv %s -o=custom-columns=:." "spec.capacity.storage" % pv_name) for w in waiter.Waiter(timeout, wait_step): pv_size = command.cmd_run(cmd, hostname=hostname).split()[0] pv_size = int(pv_size.replace("Gi", "")) if pv_size == size: g.log.info("verification of pv %s of size %d " "successful" % (pv_name, size)) return True else: g.log.info("sleeping for %s sec" % wait_step) continue err_msg = ("verification of pv %s size of %d failed -" "pv_size- %d" % (pv_name, size, pv_size)) g.log.error(err_msg) raise AssertionError(err_msg) def get_pv_name_from_pvc(hostname, pvc_name): ''' Returns PV name of the corresponding PVC name Args: hostname (str): hostname on which we want to find pv name pvc_name (str): pvc_name for which we want to find corresponding pv name Returns: pv_name (str): pv name if successful, otherwise raise Exception ''' cmd = ("oc get pvc %s -o=custom-columns=:." "spec.volumeName" % pvc_name) pv_name = command.cmd_run(cmd, hostname=hostname) g.log.info("pv name is %s for pvc %s" % ( pv_name, pvc_name)) return pv_name def get_vol_names_from_pv(hostname, pv_name): ''' Returns the heketi and gluster vol names of the corresponding PV Args: hostname (str): hostname on which we want to find vol names pv_name (str): pv_name for which we want to find corresponding vol names Returns: volname (dict): dict if successful {"heketi_vol": heketi_vol_name, "gluster_vol": gluster_vol_name ex: {"heketi_vol": " xxxx", "gluster_vol": "vol_xxxx"] otherwise raise Exception ''' vol_dict = {} cmd = ("oc get pv %s -o=custom-columns=" ":.metadata.annotations." "'gluster\.kubernetes\.io\/heketi\-volume\-id'," ":.spec.glusterfs.path" % pv_name) vol_list = command.cmd_run(cmd, hostname=hostname).split() vol_dict = {"heketi_vol": vol_list[0], "gluster_vol": vol_list[1]} g.log.info("gluster vol name is %s and heketi vol name" " is %s for pv %s" % (vol_list[1], vol_list[0], pv_name)) return vol_dict @podcmd.GlustoPod() def verify_brick_count_gluster_vol(hostname, brick_count, gluster_vol): ''' Verify brick count for gluster volume Args: hostname (str): hostname on which we want to check brick count brick_count (int): integer value to verify gluster_vol (str): gluster vol name Returns: bool: True, if successful otherwise raise Exception ''' gluster_pod = get_ocp_gluster_pod_names(hostname)[1] p = podcmd.Pod(hostname, gluster_pod) out = get_online_bricks_list(p, gluster_vol) if brick_count == len(out): g.log.info("successfully verified brick count %s " "for vol %s" % (brick_count, gluster_vol)) return True err_msg = ("verification of brick count %s for vol %s" "failed, count found %s" % ( brick_count, gluster_vol, len(out))) raise AssertionError(err_msg) @podcmd.GlustoPod() def verify_brick_status_online_gluster_vol(hostname, gluster_vol): ''' Verify if all the bricks are online for the gluster volume Args: hostname (str): hostname on which we want to check brick status gluster_vol (str): gluster vol name Returns: bool: True, if successful otherwise raise Exception ''' gluster_pod = get_ocp_gluster_pod_names(hostname)[1] p = podcmd.Pod(hostname, gluster_pod) brick_list = get_all_bricks(p, gluster_vol) if brick_list is None: error_msg = ("failed to get brick list for vol" " %s" % gluster_vol) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) out = are_bricks_online(p, gluster_vol, brick_list) if out: g.log.info("verification of brick status as online" " for gluster vol %s successful" % gluster_vol) return True error_msg = ("verification of brick status as online" " for gluster vol %s failed" % gluster_vol) g.log.error(error_msg) raise exceptions.ExecutionError(error_msg) def verify_gluster_vol_for_pvc(hostname, pvc_name): ''' Verify gluster volume has been created for the corresponding PVC Also checks if all the bricks of that gluster volume are online Args: hostname (str): hostname on which we want to find gluster vol pvc_name (str): pvc_name for which we want to find corresponding gluster vol Returns: bool: True if successful otherwise raise Exception ''' verify_pvc_status_is_bound(hostname, pvc_name) pv_name = get_pv_name_from_pvc(hostname, pvc_name) vol_dict = get_vol_names_from_pv(hostname, pv_name) gluster_vol = vol_dict["gluster_vol"] verify_brick_status_online_gluster_vol(hostname, gluster_vol) g.log.info("verification of gluster vol %s for pvc %s is" "successful" % (gluster_vol, pvc_name)) return True def get_events(hostname, obj_name=None, obj_namespace=None, obj_type=None, event_reason=None, event_type=None): """Return filtered list of events. Args: hostname (str): hostname of oc client obj_name (str): name of an object obj_namespace (str): namespace where object is located obj_type (str): type of an object, i.e. PersistentVolumeClaim or Pod event_reason (str): reason why event was created, i.e. Created, Started, Unhealthy, SuccessfulCreate, Scheduled ... event_type (str): type of an event, i.e. Normal or Warning Returns: List of dictionaries, where the latter are of following structure: { "involvedObject": { "kind": "ReplicationController", "name": "foo-object-name", "namespace": "foo-object-namespace", }, "message": "Created pod: foo-object-name", "metadata": { "creationTimestamp": "2018-10-19T18:27:09Z", "name": "foo-object-name.155f15db4e72cc2e", "namespace": "foo-object-namespace", }, "reason": "SuccessfulCreate", "reportingComponent": "", "reportingInstance": "", "source": {"component": "replication-controller"}, "type": "Normal" } """ field_selector = [] if obj_name: field_selector.append('involvedObject.name=%s' % obj_name) if obj_namespace: field_selector.append('involvedObject.namespace=%s' % obj_namespace) if obj_type: field_selector.append('involvedObject.kind=%s' % obj_type) if event_reason: field_selector.append('reason=%s' % event_reason) if event_type: field_selector.append('type=%s' % event_type) cmd = "oc get events -o yaml --field-selector %s" % ",".join( field_selector or "''") return yaml.load(command.cmd_run(cmd, hostname=hostname))['items'] def wait_for_events(hostname, obj_name=None, obj_namespace=None, obj_type=None, event_reason=None, event_type=None, timeout=120, wait_step=3): """Wait for appearence of specific set of events.""" for w in waiter.Waiter(timeout, wait_step): events = get_events( hostname=hostname, obj_name=obj_name, obj_namespace=obj_namespace, obj_type=obj_type, event_reason=event_reason, event_type=event_type) if events: return events if w.expired: err_msg = ("Exceeded %ssec timeout waiting for events." % timeout) g.log.error(err_msg) raise exceptions.ExecutionError(err_msg)