[CNS-1393-1395] Restart gluster services then validate volumes

Change-Id: I58f8628c352556ac30c31777707b52cc2aea48ac
Signed-off-by: vamahaja <vamahaja@redhat.com>

5 files changed, 441 insertions, 3 deletions

diff --git a/cns-libs/cnslibs/common/heketi_ops.py b/cns-libs/cnslibs/common/heketi_ops.py
index af021599..3025b813 100644
--- a/cns-libs/cnslibs/common/heketi_ops.py
+++ b/cns-libs/cnslibs/common/heketi_ops.py
@@ -2,15 +2,19 @@
     Description: Library for heketi operations.
 """
 
+import json
+import six
+
 from glusto.core import Glusto as g
+from glustolibs.gluster.block_ops import block_list
+from glustolibs.gluster.volume_ops import get_volume_list
 from collections import OrderedDict
-import json
 try:
     from heketi import HeketiClient
 except ImportError:
     g.log.error("Please install python-client for heketi and re-run the test")
 
-from cnslibs.common import exceptions
+from cnslibs.common import exceptions, podcmd
 
 HEKETI_SSH_KEY = "/etc/heketi/heketi_key"
 HEKETI_CONFIG_FILE = "/etc/heketi/heketi.json"
@@ -2341,3 +2345,43 @@ def rm_arbiter_tag(heketi_client_node, heketi_server_url, source, source_id,
 
     return rm_tags(heketi_client_node, heketi_server_url,
                    source, source_id, 'arbiter', **kwargs)
+
+
+@podcmd.GlustoPod()
+def match_heketi_and_gluster_block_volumes(
+        gluster_pod, heketi_block_volumes, block_vol_prefix, hostname=None):
+    """Match block volumes from heketi and gluster
+
+    Args:
+        gluster_pod (podcmd | str): gluster pod class object has gluster
+                                    pod and ocp master node or gluster
+                                    pod name
+        heketi_block_volumes (list): list of heketi block volumes with
+                                     which gluster block volumes need to
+                                     be matched
+        block_vol_prefix (str): block volume prefix by which the block
+                                volumes needs to be filtered
+        hostname (str): master node on which gluster pod exists
+
+    """
+    if isinstance(gluster_pod, podcmd.Pod):
+        g.log.info("Recieved gluster pod object using same")
+    elif isinstance(gluster_pod, six.string_types) and hostname:
+        g.log.info("Recieved gluster pod name and hostname")
+        gluster_pod = podcmd.Pod(hostname, gluster_pod)
+    else:
+        raise exceptions.ExecutionError("Invalid glsuter pod parameter")
+
+    gluster_vol_list = get_volume_list(gluster_pod)
+
+    gluster_vol_block_list = []
+    for gluster_vol in gluster_vol_list[1:]:
+        ret, out, err = block_list(gluster_pod, gluster_vol)
+        gluster_vol_block_list.extend([
+            block_vol.replace(block_vol_prefix, "")
+            for block_vol in json.loads(out)["blocks"]
+            if block_vol.startswith(block_vol_prefix)
+        ])
+
+    assert sorted(gluster_vol_block_list) == heketi_block_volumes, (
+        "Gluster and Heketi Block volume list match failed")
diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py
index 89bd68cd..9f836dae 100644
--- a/cns-libs/cnslibs/common/openshift_ops.py
+++ b/cns-libs/cnslibs/common/openshift_ops.py
@@ -29,6 +29,9 @@ from cnslibs.common.heketi_ops import (
 
 PODS_WIDE_RE = re.compile(
     '(\S+)\s+(\S+)\s+(\w+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+).*\n')
+SERVICE_STATUS = "systemctl status %s"
+SERVICE_RESTART = "systemctl restart %s"
+SERVICE_STATUS_REGEX = "Active: active \((.*)\) since .*;.*"
 
 
 def oc_get_pods(ocp_node):
@@ -473,6 +476,60 @@ def oc_delete(ocp_node, rtype, name, raise_on_absence=True):
     g.log.info('Deleted resource: %r %r', rtype, name)
 
 
+def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None,
+                           raise_on_error=True):
+    """Get an OCP resource by custom column names.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        rtype (str): Name of the resource type (pod, storageClass, etc).
+        custom (str): Name of the custom columm to fetch.
+        name (str|None): Name of the resource to fetch.
+        selector (str|list|None): Column Name or list of column
+                                  names select to.
+        raise_on_error (bool): If set to true a failure to fetch
+            resource inforation will raise an error, otherwise
+            an empty dict will be returned.
+    Returns:
+        list: List containting data about the resource custom column
+    Raises:
+        AssertionError: Raised when unable to get resource and
+            `raise_on_error` is true.
+    Example:
+        Get all "pvc" with "metadata.name" parameter values:
+            pvc_details = oc_get_custom_resource(
+                ocp_node, "pvc", ":.metadata.name"
+            )
+    """
+    cmd = ['oc', 'get', rtype, '--no-headers']
+
+    cmd.append('-o=custom-columns=%s' % (
+        ','.join(custom) if isinstance(custom, list) else custom))
+
+    if selector:
+        cmd.append('--selector %s' % (
+            ','.join(selector) if isinstance(selector, list) else selector))
+
+    if name:
+        cmd.append(name)
+
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error('Failed to get %s: %s: %r', rtype, name, err)
+        if raise_on_error:
+            raise AssertionError('failed to get %s: %s: %r'
+                                 % (rtype, name, err))
+        return []
+
+    if name:
+        return filter(None, map(str.strip, (out.strip()).split(' ')))
+    else:
+        out_list = []
+        for line in (out.strip()).split('\n'):
+            out_list.append(filter(None, map(str.strip, line.split(' '))))
+        return out_list
+
+
 def oc_get_yaml(ocp_node, rtype, name=None, raise_on_error=True):
     """Get an OCP resource by name.
 
@@ -1286,3 +1343,107 @@ def wait_for_events(hostname,
         err_msg = ("Exceeded %ssec timeout waiting for events." % timeout)
         g.log.error(err_msg)
         raise exceptions.ExecutionError(err_msg)
+
+
+def match_pvc_and_pv(hostname, prefix):
+    """Match OCP PVCs and PVs generated
+
+    Args:
+        hostname (str): hostname of oc client
+        prefix (str): pv prefix used by user at time
+                      of pvc creation
+    """
+    pvc_list = sorted([
+        pvc[0]
+        for pvc in oc_get_custom_resource(hostname, "pvc", ":.metadata.name")
+        if pvc[0].startswith(prefix)
+    ])
+
+    pv_list = sorted([
+        pv[0]
+        for pv in oc_get_custom_resource(
+            hostname, "pv", ":.spec.claimRef.name"
+        )
+        if pv[0].startswith(prefix)
+    ])
+
+    assert pvc_list == pv_list, "PVC and PV list match failed"
+
+
+def match_pv_and_heketi_block_volumes(
+        hostname, heketi_block_volumes, pvc_prefix):
+    """Match heketi block volumes and OC PVCs
+
+    Args:
+        hostname (str): hostname on which we want to check heketi
+                        block volumes and OCP PVCs
+        heketi_block_volumes (list): list of heketi block volume names
+        pvc_prefix (str): pv prefix given by user at the time of pvc creation
+    """
+    custom_columns = [
+        ':.spec.claimRef.name',
+        ':.metadata.annotations."pv\.kubernetes\.io\/provisioned\-by"',
+        ':.metadata.annotations."gluster\.org\/volume\-id"'
+    ]
+    pv_block_volumes = sorted([
+        pv[2]
+        for pv in oc_get_custom_resource(hostname, "pv", custom_columns)
+        if pv[0].startswith(pvc_prefix) and pv[1] == "gluster.org/glusterblock"
+    ])
+
+    assert pv_block_volumes == heketi_block_volumes, (
+        "PV and Heketi Block list match failed")
+
+
+def check_service_status(
+        hostname, podname, service, status, timeout=180, wait_step=3):
+    """Checks provided service to be in "Running" status for given
+       timeout on given podname
+
+    Args:
+        hostname (str): hostname on which we want to check service
+        podname (str): pod name on which service needs to be restarted
+        service (str): service which needs to be restarted
+        status (str): status to be checked
+        timeout (int): seconds to wait before service starts having
+                       specified 'status'
+        wait_step (int): interval in seconds to wait before checking
+                         service again.
+    """
+    err_msg = ("Exceeded timeout of %s sec for verifying %s service to start "
+               "having '%s' status" % (timeout, service, status))
+
+    for w in waiter.Waiter(timeout, wait_step):
+        ret, out, err = oc_rsh(hostname, podname, SERVICE_STATUS % service)
+        if ret != 0:
+            err_msg = ("failed to get service %s's status on pod %s" %
+                       (service, podname))
+            g.log.error(err_msg)
+            raise AssertionError(err_msg)
+
+        for line in out.splitlines():
+            status_match = re.search(SERVICE_STATUS_REGEX, line)
+            if status_match and status_match.group(1) == status:
+                return True
+
+    if w.expired:
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def restart_service_on_pod(hostname, podname, service):
+    """Restarts service on podname given
+
+    Args:
+        hostname (str): hostname on which we want to restart service
+        podname (str): pod name on which service needs to be restarted
+        service (str): service which needs to be restarted
+    Raises:
+        AssertionError in case failed to restarts service
+    """
+    ret, out, err = oc_rsh(hostname, podname, SERVICE_RESTART % service)
+    if ret != 0:
+        err_msg = ("failed to restart service %s on pod %s" %
+                   (service, podname))
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
diff --git a/cns-libs/setup.py b/cns-libs/setup.py
index 06f49c1d..42f1fd83 100644
--- a/cns-libs/setup.py
+++ b/cns-libs/setup.py
@@ -22,7 +22,7 @@ setup(
         'Programming Language :: Python :: 2.7'
         'Topic :: Software Development :: Testing'
     ],
-    install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff'],
+    install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff', 'six'],
     dependency_links=[
         'http://github.com/loadtheaccumulator/glusto/tarball/master#egg=glusto'
     ],
diff --git a/tests/functional/common/gluster_stability/__init__.py b/tests/functional/common/gluster_stability/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/functional/common/gluster_stability/__init__.py
diff --git a/tests/functional/common/gluster_stability/test_gluster_services_restart.py b/tests/functional/common/gluster_stability/test_gluster_services_restart.py
new file mode 100644
index 00000000..2cc09099
--- /dev/null
+++ b/tests/functional/common/gluster_stability/test_gluster_services_restart.py
@@ -0,0 +1,233 @@
+
+import ddt
+import re
+
+from cnslibs.common.heketi_ops import (
+    heketi_blockvolume_list,
+    match_heketi_and_gluster_block_volumes
+)
+from cnslibs.common.openshift_ops import (
+    check_service_status,
+    get_ocp_gluster_pod_names,
+    get_pod_name_from_dc,
+    match_pv_and_heketi_block_volumes,
+    match_pvc_and_pv,
+    oc_create_app_dc_with_io,
+    oc_create_pvc,
+    oc_create_sc,
+    oc_create_secret,
+    oc_delete,
+    oc_get_yaml,
+    restart_service_on_pod,
+    scale_dc_pod_amount_and_wait,
+    verify_pvc_status_is_bound,
+    wait_for_pod_be_ready,
+    wait_for_resource_absence
+)
+from cnslibs.cns.cns_baseclass import CnsBaseClass
+from cnslibs.common import podcmd
+
+HEKETI_BLOCK_VOLUME_REGEX = "^Id:(.*).Cluster:(.*).Name:%s_(.*)$"
+
+SERVICE_TARGET = "gluster-block-target"
+SERVICE_BLOCKD = "gluster-blockd"
+SERVICE_TCMU = "tcmu-runner"
+
+
+@ddt.ddt
+class GlusterStabilityTestSetup(CnsBaseClass):
+    """class for gluster stability (restarts different servces) testcases
+       TC No's: CNS-1393, CNS-1394, CNS-1395
+    """
+
+    def setUp(self):
+        """Deploys, Verifies and adds resources required for testcases
+           in cleanup method
+        """
+        self.oc_node = self.ocp_master_node[0]
+        self.gluster_pod = get_ocp_gluster_pod_names(self.oc_node)[0]
+
+        # prefix used to create resources, generating using glusto_test_id
+        # which uses time and date of test case
+        self.prefix = "autotest-%s" % (self.glustotest_run_id.replace("_", ""))
+
+        _cns_storage_class = self.cns_storage_class['storage_class2']
+        self.provisioner = _cns_storage_class["provisioner"]
+        self.restsecretname = _cns_storage_class["restsecretname"]
+        self.restsecretnamespace = _cns_storage_class["restsecretnamespace"]
+        self.restuser = _cns_storage_class["restuser"]
+        self.resturl = _cns_storage_class["resturl"]
+
+        _cns_secret = self.cns_secret['secret2']
+        self.secretnamespace = _cns_secret['namespace']
+        self.secrettype = _cns_secret['type']
+
+        # using pvc size count as 1 by default
+        self.pvcsize = 1
+
+        # using pvc count as 10 by default
+        self.pvccount = 10
+
+        # create gluster block storage class, PVC and user app pod
+        self.sc_name, self.pvc_name, self.dc_name, self.secret_name = (
+            self.deploy_resouces()
+        )
+
+        # verify storage class
+        oc_get_yaml(self.oc_node, "sc", self.sc_name)
+
+        # verify pod creation, it's state and get the pod name
+        self.pod_name = get_pod_name_from_dc(
+            self.oc_node, self.dc_name, timeout=180, wait_step=3
+        )
+        wait_for_pod_be_ready(
+            self.oc_node, self.pod_name, timeout=180, wait_step=3
+        )
+        verify_pvc_status_is_bound(self.oc_node, self.pvc_name)
+
+        # create pvc's to test
+        self.pvc_list = []
+        for pvc in range(self.pvccount):
+            test_pvc_name = oc_create_pvc(
+                self.oc_node, self.sc_name,
+                pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
+            )
+            self.pvc_list.append(test_pvc_name)
+            self.addCleanup(
+                wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name,
+                timeout=600, interval=10
+            )
+
+        for pvc_name in self.pvc_list:
+            self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
+
+    def deploy_resouces(self):
+        """Deploys required resources storage class, pvc and user app
+           with continous I/O runnig
+
+        Returns:
+            sc_name (str): deployed storage class name
+            pvc_name (str): deployed persistent volume claim name
+            dc_name (str): deployed deployment config name
+            secretname (str): created secret file name
+        """
+        secretname = oc_create_secret(
+            self.oc_node, namespace=self.secretnamespace,
+            data_key=self.heketi_cli_key, secret_type=self.secrettype)
+        self.addCleanup(oc_delete, self.oc_node, 'secret', secretname)
+
+        sc_name = oc_create_sc(
+            self.oc_node,
+            sc_name_prefix=self.prefix, provisioner=self.provisioner,
+            resturl=self.resturl, restuser=self.restuser,
+            restsecretnamespace=self.restsecretnamespace,
+            restsecretname=secretname, volumenameprefix=self.prefix
+        )
+        self.addCleanup(oc_delete, self.oc_node, "sc", sc_name)
+
+        pvc_name = oc_create_pvc(
+            self.oc_node, sc_name,
+            pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
+        )
+        self.addCleanup(
+            wait_for_resource_absence, self.oc_node, "pvc", pvc_name,
+            timeout=120, interval=5
+        )
+        self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)
+
+        dc_name = oc_create_app_dc_with_io(
+            self.oc_node, pvc_name, dc_name_prefix=self.prefix
+        )
+        self.addCleanup(oc_delete, self.oc_node, "dc", dc_name)
+        self.addCleanup(scale_dc_pod_amount_and_wait, self.oc_node, dc_name, 0)
+
+        return sc_name, pvc_name, dc_name, secretname
+
+    def get_heketi_block_volumes(self):
+        """lists heketi block volumes
+
+        Returns:
+            list : list of ids of heketi block volumes
+        """
+        heketi_cmd_out = heketi_blockvolume_list(
+            self.heketi_client_node,
+            self.heketi_server_url,
+            secret=self.heketi_cli_key,
+            user=self.heketi_cli_user
+        )
+
+        self.assertTrue(heketi_cmd_out, "failed to get block volume list")
+
+        heketi_block_volume_ids = []
+        heketi_block_volume_names = []
+        for block_vol in heketi_cmd_out.split("\n"):
+            heketi_vol_match = re.search(
+                HEKETI_BLOCK_VOLUME_REGEX % self.prefix, block_vol.strip()
+            )
+            if heketi_vol_match:
+                heketi_block_volume_ids.append(
+                    (heketi_vol_match.group(1)).strip()
+                )
+                heketi_block_volume_names.append(
+                    (heketi_vol_match.group(3)).strip()
+                )
+
+        return (sorted(heketi_block_volume_ids), sorted(
+            heketi_block_volume_names)
+        )
+
+    def validate_volumes_and_blocks(self):
+        """Validates PVC and block volumes generated through heketi and OCS
+        """
+
+        # verify pvc status is in "Bound" for all the pvc
+        for pvc in self.pvc_list:
+            verify_pvc_status_is_bound(
+                self.oc_node, pvc, timeout=300, wait_step=10
+            )
+
+        # validate pvcs and pvs created on OCS
+        match_pvc_and_pv(self.oc_node, self.prefix)
+
+        # get list of block volumes using heketi
+        heketi_block_volume_ids, heketi_block_volume_names = (
+            self.get_heketi_block_volumes()
+        )
+
+        # validate block volumes listed by heketi and pvs
+        match_pv_and_heketi_block_volumes(
+            self.oc_node, heketi_block_volume_ids, self.prefix
+        )
+
+        # validate block volumes listed by heketi and gluster
+        gluster_pod_obj = podcmd.Pod(self.heketi_client_node, self.gluster_pod)
+        match_heketi_and_gluster_block_volumes(
+            gluster_pod_obj, heketi_block_volume_names, "%s_" % self.prefix
+        )
+
+    @ddt.data(SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET)
+    def test_restart_services_provision_volume_and_run_io(self, service):
+        """[CNS-1393-1395] Restart gluster service then validate volumes
+        """
+        # restarts glusterfs service
+        restart_service_on_pod(self.oc_node, self.gluster_pod, service)
+
+        # wait for deployed user pod to be in Running state after restarting
+        # service
+        wait_for_pod_be_ready(
+            self.oc_node, self.pod_name, timeout=60, wait_step=5
+        )
+
+        # checks if all glusterfs services are in running state
+        for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET):
+            status = "exited" if service == SERVICE_TARGET else "running"
+            self.assertTrue(
+                check_service_status(
+                    self.oc_node, self.gluster_pod, service, status
+                ),
+                "service %s is not in %s state" % (service, status)
+            )
+
+        # validates pvc, pv, heketi block and gluster block count after
+        # service restarts
+        self.validate_volumes_and_blocks()