Reorder lib files removing redundant dir layer

Move all the files of 'cns-libs/cnslibs/common' dir
to the 'openshift-storage-libs/openshiftstoragelibs',
because 'common' is the only dir there, which doesn't really makes sense.
And "cns" is old project name, so, replace it with
"openshift-storage-libs".

Also, fix all the imports of these libs.

Change-Id: Ife00a73554e73b21b214b15016b0c8dbbf423446

16 files changed, 4600 insertions, 0 deletions

diff --git a/openshift-storage-libs/MANIFEST.in b/openshift-storage-libs/MANIFEST.in
new file mode 100644
index 00000000..121de5bf
--- /dev/null
+++ b/openshift-storage-libs/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include openshiftstoragelibs *.yaml *.json *.txt
diff --git a/openshift-storage-libs/openshiftstoragelibs/__init__.py b/openshift-storage-libs/openshiftstoragelibs/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/__init__.py
diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
new file mode 100644
index 00000000..366af6a9
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
@@ -0,0 +1,318 @@
+import datetime
+import unittest
+
+from glusto.core import Glusto as g
+
+from openshiftstoragelibs import command
+from openshiftstoragelibs.exceptions import (
+    ConfigError,
+    ExecutionError,
+)
+from openshiftstoragelibs.heketi_ops import (
+    hello_heketi,
+    heketi_blockvolume_delete,
+    heketi_volume_delete,
+)
+from openshiftstoragelibs.openshift_ops import (
+    get_pod_name_from_dc,
+    get_pv_name_from_pvc,
+    oc_create_app_dc_with_io,
+    oc_create_pvc,
+    oc_create_sc,
+    oc_create_secret,
+    oc_delete,
+    oc_get_custom_resource,
+    scale_dc_pod_amount_and_wait,
+    switch_oc_project,
+    verify_pvc_status_is_bound,
+    wait_for_pod_be_ready,
+    wait_for_resource_absence,
+)
+
+
+class BaseClass(unittest.TestCase):
+    """Base class for test classes."""
+    ERROR_OR_FAILURE_EXISTS = False
+    STOP_ON_FIRST_FAILURE = bool(g.config.get("common", {}).get(
+        "stop_on_first_failure", False))
+
+    @classmethod
+    def setUpClass(cls):
+        """Initialize all the variables necessary for test cases."""
+        super(BaseClass, cls).setUpClass()
+
+        # Initializes OCP config variables
+        cls.ocp_servers_info = g.config['ocp_servers']
+        cls.ocp_master_node = g.config['ocp_servers']['master'].keys()
+        cls.ocp_master_node_info = g.config['ocp_servers']['master']
+        cls.ocp_client = g.config['ocp_servers']['client'].keys()
+        cls.ocp_client_info = g.config['ocp_servers']['client']
+        cls.ocp_nodes = g.config['ocp_servers']['nodes'].keys()
+        cls.ocp_nodes_info = g.config['ocp_servers']['nodes']
+
+        # Initializes storage project config variables
+        openshift_config = g.config.get("cns", g.config.get("openshift"))
+        cls.storage_project_name = openshift_config.get(
+            'storage_project_name',
+            openshift_config.get('setup', {}).get('cns_project_name'))
+
+        # Initializes heketi config variables
+        heketi_config = openshift_config['heketi_config']
+        cls.heketi_dc_name = heketi_config['heketi_dc_name']
+        cls.heketi_service_name = heketi_config['heketi_service_name']
+        cls.heketi_client_node = heketi_config['heketi_client_node']
+        cls.heketi_server_url = heketi_config['heketi_server_url']
+        cls.heketi_cli_user = heketi_config['heketi_cli_user']
+        cls.heketi_cli_key = heketi_config['heketi_cli_key']
+
+        cls.gluster_servers = g.config['gluster_servers'].keys()
+        cls.gluster_servers_info = g.config['gluster_servers']
+
+        cls.storage_classes = openshift_config['dynamic_provisioning'][
+            'storage_classes']
+        cls.sc = cls.storage_classes.get(
+            'storage_class1', cls.storage_classes.get('file_storage_class'))
+        cmd = "echo -n %s | base64" % cls.heketi_cli_key
+        ret, out, err = g.run(cls.ocp_master_node[0], cmd, "root")
+        if ret != 0:
+            raise ExecutionError("failed to execute cmd %s on %s out: %s "
+                                 "err: %s" % (
+                                     cmd, cls.ocp_master_node[0], out, err))
+        cls.secret_data_key = out.strip()
+
+        # Checks if heketi server is alive
+        if not hello_heketi(cls.heketi_client_node, cls.heketi_server_url):
+            raise ConfigError("Heketi server %s is not alive"
+                              % cls.heketi_server_url)
+
+        # Switch to the storage project
+        if not switch_oc_project(
+                cls.ocp_master_node[0], cls.storage_project_name):
+            raise ExecutionError("Failed to switch oc project on node %s"
+                                 % cls.ocp_master_node[0])
+
+        if 'glustotest_run_id' not in g.config:
+            g.config['glustotest_run_id'] = (
+                datetime.datetime.now().strftime('%H_%M_%d_%m_%Y'))
+        cls.glustotest_run_id = g.config['glustotest_run_id']
+        msg = "Setupclass: %s : %s" % (cls.__name__, cls.glustotest_run_id)
+        g.log.info(msg)
+
+    def setUp(self):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and
+                BaseClass.ERROR_OR_FAILURE_EXISTS):
+            self.skipTest("Test is skipped, because of the restriction "
+                          "to one test case failure.")
+
+        super(BaseClass, self).setUp()
+
+        msg = "Starting Test : %s : %s" % (self.id(), self.glustotest_run_id)
+        g.log.info(msg)
+
+    def tearDown(self):
+        super(BaseClass, self).tearDown()
+        msg = "Ending Test: %s : %s" % (self.id(), self.glustotest_run_id)
+        g.log.info(msg)
+
+    @classmethod
+    def tearDownClass(cls):
+        super(BaseClass, cls).tearDownClass()
+        msg = "Teardownclass: %s : %s" % (cls.__name__, cls.glustotest_run_id)
+        g.log.info(msg)
+
+    def cmd_run(self, cmd, hostname=None, raise_on_error=True):
+        if not hostname:
+            hostname = self.ocp_master_node[0]
+        return command.cmd_run(
+            cmd=cmd, hostname=hostname, raise_on_error=raise_on_error)
+
+    def create_secret(self, secret_name_prefix="autotests-secret-"):
+        secret_name = oc_create_secret(
+            self.ocp_client[0],
+            secret_name_prefix=secret_name_prefix,
+            namespace=(self.sc.get(
+                'secretnamespace',
+                self.sc.get('restsecretnamespace', 'default'))),
+            data_key=self.heketi_cli_key,
+            secret_type=self.sc.get('provisioner', 'kubernetes.io/glusterfs'))
+        self.addCleanup(
+            oc_delete, self.ocp_client[0], 'secret', secret_name)
+        return secret_name
+
+    def create_storage_class(self, secret_name=None,
+                             sc_name_prefix="autotests-sc",
+                             create_vol_name_prefix=False,
+                             allow_volume_expansion=False,
+                             reclaim_policy="Delete",
+                             set_hacount=None,
+                             is_arbiter_vol=False, arbiter_avg_file_size=None):
+
+        # Create secret if one is not specified
+        if not secret_name:
+            secret_name = self.create_secret()
+
+        # Create storage class
+        secret_name_option = "secretname"
+        secret_namespace_option = "secretnamespace"
+        provisioner = self.sc.get("provisioner", "kubernetes.io/glusterfs")
+        if provisioner != "kubernetes.io/glusterfs":
+            secret_name_option = "rest%s" % secret_name_option
+            secret_namespace_option = "rest%s" % secret_namespace_option
+        parameters = {
+            "resturl": self.sc["resturl"],
+            "restuser": self.sc["restuser"],
+            secret_name_option: secret_name,
+            secret_namespace_option: self.sc.get(
+                "secretnamespace", self.sc.get("restsecretnamespace")),
+        }
+        if set_hacount:
+            parameters["hacount"] = self.sc.get("hacount", "3")
+        if is_arbiter_vol:
+            parameters["volumeoptions"] = "user.heketi.arbiter true"
+            if arbiter_avg_file_size:
+                parameters["volumeoptions"] += (
+                    ",user.heketi.average-file-size %s" % (
+                        arbiter_avg_file_size))
+        if create_vol_name_prefix:
+            parameters["volumenameprefix"] = self.sc.get(
+                "volumenameprefix", "autotest")
+        self.sc_name = oc_create_sc(
+            self.ocp_client[0],
+            sc_name_prefix=sc_name_prefix,
+            provisioner=provisioner,
+            allow_volume_expansion=allow_volume_expansion,
+            reclaim_policy=reclaim_policy,
+            **parameters)
+        self.addCleanup(oc_delete, self.ocp_client[0], "sc", self.sc_name)
+        return self.sc_name
+
+    def create_and_wait_for_pvcs(self, pvc_size=1,
+                                 pvc_name_prefix="autotests-pvc",
+                                 pvc_amount=1, sc_name=None,
+                                 timeout=120, wait_step=3):
+        node = self.ocp_client[0]
+
+        # Create storage class if not specified
+        if not sc_name:
+            if getattr(self, "sc_name", ""):
+                sc_name = self.sc_name
+            else:
+                sc_name = self.create_storage_class()
+
+        # Create PVCs
+        pvc_names = []
+        for i in range(pvc_amount):
+            pvc_name = oc_create_pvc(
+                node, sc_name, pvc_name_prefix=pvc_name_prefix,
+                pvc_size=pvc_size)
+            pvc_names.append(pvc_name)
+            self.addCleanup(
+                wait_for_resource_absence, node, 'pvc', pvc_name)
+
+        # Wait for PVCs to be in bound state
+        try:
+            for pvc_name in pvc_names:
+                verify_pvc_status_is_bound(node, pvc_name, timeout, wait_step)
+        finally:
+            reclaim_policy = oc_get_custom_resource(
+                node, 'sc', ':.reclaimPolicy', sc_name)[0]
+
+            for pvc_name in pvc_names:
+                if reclaim_policy == 'Retain':
+                    pv_name = get_pv_name_from_pvc(node, pvc_name)
+                    self.addCleanup(oc_delete, node, 'pv', pv_name,
+                                    raise_on_absence=False)
+                    custom = (r':.metadata.annotations."gluster\.kubernetes'
+                              r'\.io\/heketi\-volume\-id"')
+                    vol_id = oc_get_custom_resource(
+                        node, 'pv', custom, pv_name)[0]
+                    if self.sc.get('provisioner') == "kubernetes.io/glusterfs":
+                        self.addCleanup(heketi_volume_delete,
+                                        self.heketi_client_node,
+                                        self.heketi_server_url, vol_id,
+                                        raise_on_error=False)
+                    else:
+                        self.addCleanup(heketi_blockvolume_delete,
+                                        self.heketi_client_node,
+                                        self.heketi_server_url, vol_id,
+                                        raise_on_error=False)
+                self.addCleanup(oc_delete, node, 'pvc', pvc_name,
+                                raise_on_absence=False)
+
+        return pvc_names
+
+    def create_and_wait_for_pvc(self, pvc_size=1,
+                                pvc_name_prefix='autotests-pvc', sc_name=None):
+        self.pvc_name = self.create_and_wait_for_pvcs(
+            pvc_size=pvc_size, pvc_name_prefix=pvc_name_prefix, sc_name=sc_name
+        )[0]
+        return self.pvc_name
+
+    def create_dc_with_pvc(self, pvc_name, timeout=300, wait_step=10):
+        dc_name = oc_create_app_dc_with_io(self.ocp_client[0], pvc_name)
+        self.addCleanup(oc_delete, self.ocp_client[0], 'dc', dc_name)
+        self.addCleanup(
+            scale_dc_pod_amount_and_wait, self.ocp_client[0], dc_name, 0)
+        pod_name = get_pod_name_from_dc(self.ocp_client[0], dc_name)
+        wait_for_pod_be_ready(self.ocp_client[0], pod_name,
+                              timeout=timeout, wait_step=wait_step)
+        return dc_name, pod_name
+
+    def _is_error_or_failure_exists(self):
+        if hasattr(self, '_outcome'):
+            # Python 3.4+
+            result = self.defaultTestResult()
+            self._feedErrorsToResult(result, self._outcome.errors)
+        else:
+            # Python 2.7-3.3
+            result = getattr(
+                self, '_outcomeForDoCleanups', self._resultForDoCleanups)
+        ok_result = True
+        for attr in ('errors', 'failures'):
+            if not hasattr(result, attr):
+                continue
+            exc_list = getattr(result, attr)
+            if exc_list and exc_list[-1][0] is self:
+                ok_result = ok_result and not exc_list[-1][1]
+        if hasattr(result, '_excinfo'):
+            ok_result = ok_result and not result._excinfo
+        if ok_result:
+            return False
+        self.ERROR_OR_FAILURE_EXISTS = True
+        BaseClass.ERROR_OR_FAILURE_EXISTS = True
+        return True
+
+    def doCleanups(self):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and (
+                self.ERROR_OR_FAILURE_EXISTS or
+                self._is_error_or_failure_exists())):
+            while self._cleanups:
+                (func, args, kwargs) = self._cleanups.pop()
+                msg = ("Found test case failure. Avoiding run of scheduled "
+                       "following cleanup:\nfunc = %s\nargs = %s\n"
+                       "kwargs = %s" % (func, args, kwargs))
+                g.log.warn(msg)
+        return super(BaseClass, self).doCleanups()
+
+    @classmethod
+    def doClassCleanups(cls):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and
+                BaseClass.ERROR_OR_FAILURE_EXISTS):
+            while cls._class_cleanups:
+                (func, args, kwargs) = cls._class_cleanups.pop()
+                msg = ("Found test case failure. Avoiding run of scheduled "
+                       "following cleanup:\nfunc = %s\nargs = %s\n"
+                       "kwargs = %s" % (func, args, kwargs))
+                g.log.warn(msg)
+        return super(BaseClass, cls).doClassCleanups()
+
+
+class GlusterBlockBaseClass(BaseClass):
+    """Base class for gluster-block test cases."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Initialize all the variables necessary for test cases."""
+        super(GlusterBlockBaseClass, cls).setUpClass()
+        cls.sc = cls.storage_classes.get(
+            'storage_class2', cls.storage_classes.get('block_storage_class'))
diff --git a/openshift-storage-libs/openshiftstoragelibs/command.py b/openshift-storage-libs/openshiftstoragelibs/command.py
new file mode 100644
index 00000000..06912915
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/command.py
@@ -0,0 +1,23 @@
+from glusto.core import Glusto as g
+
+
+def cmd_run(cmd, hostname, raise_on_error=True):
+    """Glusto's command runner wrapper.
+
+    Args:
+        cmd (str): Shell command to run on the specified hostname.
+        hostname (str): hostname where Glusto should run specified command.
+        raise_on_error (bool): defines whether we should raise exception
+                               in case command execution failed.
+    Returns:
+        str: Stripped shell command's stdout value if not None.
+    """
+    ret, out, err = g.run(hostname, cmd, "root")
+    if raise_on_error:
+        msg = ("Failed to execute command '%s' on '%s' node. Got non-zero "
+               "return code '%s'. Err: %s" % (cmd, hostname, ret, err))
+        assert int(ret) == 0, msg
+
+    out = out.strip() if out else out
+
+    return out
diff --git a/openshift-storage-libs/openshiftstoragelibs/exceptions.py b/openshift-storage-libs/openshiftstoragelibs/exceptions.py
new file mode 100644
index 00000000..44daee12
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/exceptions.py
@@ -0,0 +1,23 @@
+class ConfigError(Exception):
+    '''
+    Custom exception thrown when there is an unrecoverable configuration error.
+    For example, a required configuration key is not found.
+    '''
+
+
+class ExecutionError(Exception):
+    '''
+    Custom exception thrown when a command executed by Glusto results in an
+    unrecoverable error.
+
+    For example, all hosts are not in peer state or a volume cannot be setup.
+    '''
+
+
+class NotSupportedException(Exception):
+    '''
+    Custom exception thrown when we do not support a particular feature in
+    particular product version
+
+    For example, pv resize is not supported in OCP version < 3.9
+    '''
diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
new file mode 100644
index 00000000..8ac95d82
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
@@ -0,0 +1,260 @@
+import time
+import json
+import re
+
+from glusto.core import Glusto as g
+from glustolibs.gluster.block_ops import block_list
+from glustolibs.gluster.heal_libs import is_heal_complete
+from glustolibs.gluster.volume_ops import (
+    get_volume_status,
+    get_volume_list,
+    volume_status,
+    volume_start,
+    volume_stop,
+)
+
+from openshiftstoragelibs import exceptions
+from openshiftstoragelibs.heketi_ops import heketi_blockvolume_info
+from openshiftstoragelibs.openshift_ops import cmd_run_on_gluster_pod_or_node
+from openshiftstoragelibs import podcmd
+from openshiftstoragelibs import waiter
+
+
+@podcmd.GlustoPod()
+def wait_to_heal_complete(timeout=300, wait_step=5):
+    """Monitors heal for volumes on gluster"""
+    gluster_vol_list = get_volume_list("auto_get_gluster_endpoint")
+    if not gluster_vol_list:
+        raise AssertionError("failed to get gluster volume list")
+
+    _waiter = waiter.Waiter(timeout=timeout, interval=wait_step)
+    for gluster_vol in gluster_vol_list:
+        for w in _waiter:
+            if is_heal_complete("auto_get_gluster_endpoint", gluster_vol):
+                break
+
+    if w.expired:
+        err_msg = ("reached timeout waiting for all the gluster volumes "
+                   "to reach the 'healed' state.")
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
+
+
+@podcmd.GlustoPod()
+def get_gluster_vol_status(file_vol):
+    """Get Gluster vol hosting nodes.
+
+    Args:
+        file_vol (str): file volume name.
+    """
+    # Get Gluster vol info
+    gluster_volume_status = get_volume_status(
+        "auto_get_gluster_endpoint", file_vol)
+    if not gluster_volume_status:
+        raise AssertionError("Failed to get volume status for gluster "
+                             "volume '%s'" % file_vol)
+    if file_vol in gluster_volume_status:
+        gluster_volume_status = gluster_volume_status.get(file_vol)
+    return gluster_volume_status
+
+
+@podcmd.GlustoPod()
+def get_gluster_vol_hosting_nodes(file_vol):
+    """Get Gluster vol hosting nodes.
+
+    Args:
+        file_vol (str): file volume name.
+    """
+    vol_status = get_gluster_vol_status(file_vol)
+    g_nodes = []
+    for g_node, g_node_data in vol_status.items():
+        for process_name, process_data in g_node_data.items():
+            if not process_name.startswith("/var"):
+                continue
+            g_nodes.append(g_node)
+    return g_nodes
+
+
+@podcmd.GlustoPod()
+def restart_gluster_vol_brick_processes(ocp_client_node, file_vol,
+                                        gluster_nodes):
+    """Restarts brick process of a file volume.
+
+    Args:
+        ocp_client_node (str): Node to execute OCP commands on.
+        file_vol (str): file volume name.
+        gluster_nodes (str/list): One or several IPv4 addresses of Gluster
+            nodes, where 'file_vol' brick processes must be recreated.
+    """
+    if not isinstance(gluster_nodes, (list, set, tuple)):
+        gluster_nodes = [gluster_nodes]
+
+    # Get Gluster vol brick PIDs
+    gluster_volume_status = get_gluster_vol_status(file_vol)
+    pids = ()
+    for gluster_node in gluster_nodes:
+        pid = None
+        for g_node, g_node_data in gluster_volume_status.items():
+            if g_node != gluster_node:
+                continue
+            for process_name, process_data in g_node_data.items():
+                if not process_name.startswith("/var"):
+                    continue
+                pid = process_data["pid"]
+                # When birck is down, pid of the brick is returned as -1.
+                # Which is unexepeted situation. So, add appropriate assertion.
+                assert pid != "-1", (
+                    "Got unexpected PID (-1) for '%s' gluster vol on '%s' "
+                    "node." % file_vol, gluster_node)
+        assert pid, ("Could not find 'pid' in Gluster vol data for '%s' "
+                     "Gluster node. Data: %s" % (
+                         gluster_node, gluster_volume_status))
+        pids.append((gluster_node, pid))
+
+    # Restart Gluster vol brick processes using found PIDs
+    for gluster_node, pid in pids:
+        cmd = "kill -9 %s" % pid
+        cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node)
+
+    # Wait for Gluster vol brick processes to be recreated
+    for gluster_node, pid in pids:
+        killed_pid_cmd = "ps -eaf | grep %s | grep -v grep | awk '{print $2}'"
+        _waiter = waiter.Waiter(timeout=60, interval=2)
+        for w in _waiter:
+            result = cmd_run_on_gluster_pod_or_node(
+                ocp_client_node, killed_pid_cmd, gluster_node)
+            if result.strip() == pid:
+                continue
+            g.log.info("Brick process '%s' was killed successfully on '%s'" % (
+                pid, gluster_node))
+            break
+        if w.expired:
+            error_msg = ("Process ID '%s' still exists on '%s' after waiting "
+                         "for it 60 seconds to get killed." % (
+                            pid, gluster_node))
+            g.log.error(error_msg)
+            raise exceptions.ExecutionError(error_msg)
+
+    # Start volume after gluster vol brick processes recreation
+    ret, out, err = volume_start(
+        "auto_get_gluster_endpoint", file_vol, force=True)
+    if ret != 0:
+        err_msg = "Failed to start gluster volume %s on %s. error: %s" % (
+            file_vol, gluster_node, err)
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
+
+
+@podcmd.GlustoPod()
+def restart_file_volume(file_vol, sleep_time=120):
+    """Restars file volume service.
+
+    Args:
+        file_vol (str): name of a file volume
+    """
+    gluster_volume_status = get_volume_status(
+        "auto_get_gluster_endpoint", file_vol)
+    if not gluster_volume_status:
+        raise AssertionError("failed to get gluster volume status")
+
+    g.log.info("Gluster volume %s status\n%s : " % (
+        file_vol, gluster_volume_status)
+    )
+
+    ret, out, err = volume_stop("auto_get_gluster_endpoint", file_vol)
+    if ret != 0:
+        err_msg = "Failed to stop gluster volume %s. error: %s" % (
+            file_vol, err)
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
+
+    # Explicit wait to stop ios and pvc creation for 2 mins
+    time.sleep(sleep_time)
+
+    ret, out, err = volume_start(
+        "auto_get_gluster_endpoint", file_vol, force=True)
+    if ret != 0:
+        err_msg = "failed to start gluster volume %s error: %s" % (
+            file_vol, err)
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
+
+    ret, out, err = volume_status("auto_get_gluster_endpoint", file_vol)
+    if ret != 0:
+        err_msg = ("Failed to get status for gluster volume %s error: %s" % (
+            file_vol, err))
+        g.log.error(err_msg)
+        raise AssertionError(err_msg)
+
+
+@podcmd.GlustoPod()
+def match_heketi_and_gluster_block_volumes_by_prefix(
+        heketi_block_volumes, block_vol_prefix):
+    """Match block volumes from heketi and gluster. This function can't
+       be used for block volumes with custom prefixes
+
+    Args:
+        heketi_block_volumes (list): list of heketi block volumes with
+                                     which gluster block volumes need to
+                                     be matched
+        block_vol_prefix (str): block volume prefix by which the block
+                                volumes needs to be filtered
+    """
+    gluster_vol_list = get_volume_list("auto_get_gluster_endpoint")
+
+    gluster_vol_block_list = []
+    for gluster_vol in gluster_vol_list[1:]:
+        ret, out, err = block_list("auto_get_gluster_endpoint", gluster_vol)
+        try:
+            if ret != 0 and json.loads(out)["RESULT"] == "FAIL":
+                msg = "failed to get block volume list with error: %s" % err
+                g.log.error(msg)
+                raise AssertionError(msg)
+        except Exception as e:
+            g.log.error(e)
+            raise
+
+        gluster_vol_block_list.extend([
+            block_vol.replace(block_vol_prefix, "")
+            for block_vol in json.loads(out)["blocks"]
+            if block_vol.startswith(block_vol_prefix)
+        ])
+
+    if cmp(sorted(gluster_vol_block_list), heketi_block_volumes) != 0:
+        err_msg = "Gluster and Heketi Block volume list match failed"
+        err_msg += "\nGluster Volumes: %s, " % gluster_vol_block_list
+        err_msg += "\nBlock volumes %s" % heketi_block_volumes
+        err_msg += "\nDifference: %s" % (set(gluster_vol_block_list) ^
+                                         set(heketi_block_volumes))
+        raise AssertionError(err_msg)
+
+
+@podcmd.GlustoPod()
+def get_block_hosting_volume_name(heketi_client_node, heketi_server_url,
+                                  block_volume):
+    """Returns block hosting volume name of given block volume
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        block_volume (str): Block volume of which block hosting volume
+                            returned
+    Returns:
+        str : Name of the block hosting volume for given block volume
+    """
+    block_vol_info = heketi_blockvolume_info(
+        heketi_client_node, heketi_server_url, block_volume
+    )
+
+    for line in block_vol_info.splitlines():
+        block_hosting_vol_match = re.search(
+            "^Block Hosting Volume: (.*)$", line
+        )
+
+        if not block_hosting_vol_match:
+            continue
+
+        gluster_vol_list = get_volume_list("auto_get_gluster_endpoint")
+        for vol in gluster_vol_list:
+            if block_hosting_vol_match.group(1).strip() in vol:
+                return vol
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
new file mode 100644
index 00000000..02fefe66
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
@@ -0,0 +1,1516 @@
+import json
+
+from glusto.core import Glusto as g
+
+from openshiftstoragelibs import exceptions
+from openshiftstoragelibs import heketi_version
+from openshiftstoragelibs.utils import parse_prometheus_data
+
+
+def _set_heketi_global_flags(heketi_server_url, **kwargs):
+    """Helper function to set heketi-cli global flags."""
+
+    heketi_server_url = (heketi_server_url if heketi_server_url else ("http:" +
+                         "//heketi-storage-project.cloudapps.mystorage.com"))
+    json = kwargs.get("json")
+    secret = kwargs.get("secret")
+    user = kwargs.get("user")
+    json_arg = "--json" if json else ""
+    secret_arg = "--secret %s" % secret if secret else ""
+    user_arg = "--user %s" % user if user else ""
+    if not user_arg:
+        openshift_config = g.config.get("cns", g.config.get("openshift"))
+        heketi_cli_user = openshift_config['heketi_config']['heketi_cli_user']
+        if heketi_cli_user:
+            user_arg = "--user %s" % heketi_cli_user
+            heketi_cli_key = openshift_config[
+                'heketi_config']['heketi_cli_key']
+            if heketi_cli_key is not None:
+                secret_arg = "--secret '%s'" % heketi_cli_key
+
+    return (heketi_server_url, json_arg, secret_arg, user_arg)
+
+
+def heketi_volume_create(heketi_client_node, heketi_server_url, size,
+                         raw_cli_output=False, **kwargs):
+    """Creates heketi volume with the given user options.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        size (str): Volume size
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - block : (bool)
+            - clusters : (str)|None
+            - disperse_data : (int)|None
+            - durability : (str)|None
+            - gid : (int)|None
+            - gluster_volume_options : (str)|None
+            - name : (str)|None
+            - persistent_volume : (bool)
+            - persistent_volume_endpoint : (str)|None
+            - persistent_volume_file : (str)|None
+            - redundancy : (int):None
+            - replica : (int)|None
+            - size : (int):None
+            - snapshot-factor : (float)|None
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: volume create info on success, only cli option is specified
+            without --json option, then it returns raw string output.
+        Tuple (ret, out, err): if raw_cli_output is True.
+    Raises:
+        exceptions.ExecutionError when error occurs and raw_cli_output is False
+
+    Example:
+        heketi_volume_create(heketi_client_node, heketi_server_url, size)
+    """
+
+    if not kwargs.get('user'):
+        openshift_config = g.config.get("cns", g.config.get("openshift"))
+        heketi_cli_user = openshift_config['heketi_config']['heketi_cli_user']
+        if heketi_cli_user:
+            kwargs['user'] = heketi_cli_user
+            heketi_cli_key = openshift_config[
+                'heketi_config']['heketi_cli_key']
+            if heketi_cli_key is not None:
+                kwargs['secret'] = heketi_cli_key
+
+    heketi_server_url = (heketi_server_url if heketi_server_url else ("http:" +
+                         "//heketi-storage-project.cloudapps.mystorage.com"))
+
+    block_arg = "--block" if kwargs.get("block") else ""
+    clusters_arg = ("--clusters %s" % kwargs.get("clusters")
+                    if kwargs.get("clusters") else "")
+    disperse_data_arg = ("--disperse-data %d" % kwargs.get("disperse_data")
+                         if kwargs.get("disperse_data") else "")
+    durability_arg = ("--durability %s" % kwargs.get("durability")
+                      if kwargs.get("durability") else "")
+    gid_arg = "--gid %d" % int(kwargs.get("gid")) if kwargs.get("gid") else ""
+    gluster_volume_options_arg = ("--gluster-volume-options '%s'"
+                                  % kwargs.get("gluster_volume_options")
+                                  if kwargs.get("gluster_volume_options")
+                                  else "")
+    name_arg = "--name %s" % kwargs.get("name") if kwargs.get("name") else ""
+    persistent_volume_arg = ("--persistent-volume %s"
+                             % kwargs.get("persistent_volume")
+                             if kwargs.get("persistent_volume") else "")
+    persistent_volume_endpoint_arg = ("--persistent-volume-endpoint %s"
+                                      % (kwargs.get(
+                                         "persistent_volume_endpoint"))
+                                      if (kwargs.get(
+                                          "persistent_volume_endpoint"))
+                                      else "")
+    persistent_volume_file_arg = ("--persistent-volume-file %s"
+                                  % kwargs.get("persistent_volume_file")
+                                  if kwargs.get("persistent_volume_file")
+                                  else "")
+    redundancy_arg = ("--redundancy %d" % int(kwargs.get("redundancy"))
+                      if kwargs.get("redundancy") else "")
+    replica_arg = ("--replica %d" % int(kwargs.get("replica"))
+                   if kwargs.get("replica") else "")
+    snapshot_factor_arg = ("--snapshot-factor %f"
+                           % float(kwargs.get("snapshot_factor"))
+                           if kwargs.get("snapshot_factor") else "")
+    json_arg = "--json" if kwargs.get("json") else ""
+    secret_arg = (
+        "--secret %s" % kwargs.get("secret") if kwargs.get("secret") else "")
+    user_arg = "--user %s" % kwargs.get("user") if kwargs.get("user") else ""
+
+    err_msg = "Failed to create volume. "
+
+    cmd = ("heketi-cli -s %s volume create --size=%s %s %s %s %s %s %s "
+           "%s %s %s %s %s %s %s %s %s %s" % (
+               heketi_server_url, str(size), block_arg, clusters_arg,
+               disperse_data_arg, durability_arg, gid_arg,
+               gluster_volume_options_arg, name_arg,
+               persistent_volume_arg, persistent_volume_endpoint_arg,
+               persistent_volume_file_arg, redundancy_arg, replica_arg,
+               snapshot_factor_arg, json_arg, secret_arg, user_arg))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        err_msg += "Out: %s \n Err: %s" % (out, err)
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_volume_info(heketi_client_node, heketi_server_url, volume_id,
+                       raw_cli_output=False, **kwargs):
+    """Executes heketi volume info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        volume_id (str): Volume ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: volume info on success
+        False: in case of failure
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_volume_info(heketi_client_node, volume_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s volume info %s %s %s %s" % (
+        heketi_server_url, volume_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id,
+                         expand_size, raw_cli_output=False, **kwargs):
+    """Executes heketi volume expand command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        volume_id (str): Volume ID
+        expand_size (str): volume expand size
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: volume expand info on success, only cli option is specified
+            without --json option, then it returns raw string output.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_volume_expand(heketi_client_node, heketi_server_url, volume_id,
+                             expand_size)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = ("heketi-cli -s %s volume expand --volume=%s "
+           "--expand-size=%s %s %s %s" % (
+               heketi_server_url, volume_id, expand_size, json_arg,
+               admin_key, user))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id,
+                         raw_cli_output=False, raise_on_error=True, **kwargs):
+    """Executes heketi volume delete command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        volume_id (str): Volume ID
+        raise_on_error (bool): whether or not to raise exception
+            in case of an error.
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: volume delete command output on success
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError when error occurs and raw_cli_output is False
+
+    Example:
+        heketi_volume_delete(heketi_client_node, heketi_server_url, volume_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    err_msg = "Failed to delete '%s' volume. " % volume_id
+
+    cmd = "heketi-cli -s %s volume delete %s %s %s %s" % (
+        heketi_server_url, volume_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        err_msg += "Out: %s, \nErr: %s" % (out, err)
+        g.log.error(err_msg)
+        if raise_on_error:
+            raise exceptions.ExecutionError(err_msg)
+    return out
+
+
+def heketi_volume_list(heketi_client_node, heketi_server_url,
+                       raw_cli_output=False, **kwargs):
+    """Executes heketi volume list command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: volume list with --json on success, if cli option is specified
+            without --json option or with url, it returns raw string output.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_volume_info(heketi_client_node, heketi_server_url)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s volume list %s %s %s" % (
+        heketi_server_url, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_topology_info(heketi_client_node, heketi_server_url,
+                         raw_cli_output=False, **kwargs):
+    """Executes heketi topology info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: topology info if --json option is specified. If only cli option
+            is specified, raw command output is returned on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_topology_info(heketi_client_node, heketi_server_url)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s topology info %s %s %s" % (
+        heketi_server_url, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def hello_heketi(heketi_client_node, heketi_server_url, **kwargs):
+    """Executes curl command to check if heketi server is alive.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        bool: True, if heketi server is alive
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        hello_heketi(heketi_client_node, heketi_server_url)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "curl --max-time 10 %s/hello" % heketi_server_url
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return True
+
+
+def heketi_cluster_delete(heketi_client_node, heketi_server_url, cluster_id,
+                          **kwargs):
+    """Executes heketi cluster delete command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        cluster_id (str): Cluster ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: cluster delete command output on success
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_cluster_delete(heketi_client_node, heketi_server_url,
+                              cluster_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s cluster delete %s %s %s %s" % (
+        heketi_server_url, cluster_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_cluster_info(heketi_client_node, heketi_server_url, cluster_id,
+                        **kwargs):
+    """Executes heketi cluster info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        cluster_id (str): Volume ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: cluster info on success
+        False: in case of failure
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_cluster_info(heketi_client_node, volume_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s cluster info %s %s %s %s" % (
+        heketi_server_url, cluster_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_cluster_list(heketi_client_node, heketi_server_url, **kwargs):
+    """Executes heketi cluster list command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: cluster list with --json on success, if cli option is specified
+            without --json option or with url, it returns raw string output.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_cluster_info(heketi_client_node, heketi_server_url)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s cluster list %s %s %s" % (
+        heketi_server_url, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_device_add(heketi_client_node, heketi_server_url, device_name,
+                      node_id, raw_cli_output=False, **kwargs):
+    """Executes heketi device add command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device name (str): Device name to add
+        node_id (str): Node id to add the device
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi device add command output on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_add(heketi_client_node, heketi_server_url, device_name,
+                          node_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s device add --name=%s --node=%s %s %s %s" % (
+        heketi_server_url, device_name, node_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_device_delete(heketi_client_node, heketi_server_url, device_id,
+                         raw_cli_output=False, **kwargs):
+    """Executes heketi device delete command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device id (str): Device id to delete
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi device delete command output on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_delete(heketi_client_node, heketi_server_url, device_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s device delete %s %s %s %s" % (
+        heketi_server_url, device_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_device_disable(heketi_client_node, heketi_server_url, device_id,
+                          raw_cli_output=False, **kwargs):
+    """Executes heketi device disable command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device_id (str): Device id to disable device
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi device disable command output on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_disable(heketi_client_node, heketi_server_url, device_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    cmd = "heketi-cli -s %s device disable %s %s %s %s" % (
+        heketi_server_url, device_id, json_arg, admin_key, user)
+
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_device_enable(heketi_client_node, heketi_server_url, device_id,
+                         raw_cli_output=False, **kwargs):
+    """Executes heketi device enable command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device_id (str): Device id to enable device
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi device enable command output on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_enable(heketi_client_node, heketi_server_url, device_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    cmd = "heketi-cli -s %s device enable %s %s %s %s" % (
+        heketi_server_url, device_id, json_arg, admin_key, user)
+
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_device_info(heketi_client_node, heketi_server_url, device_id,
+                       raw_cli_output=False, **kwargs):
+    """Executes heketi device info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device_id (str): Device ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        Str: device info as raw CLI output if "json" arg is not provided.
+        Dict: device info parsed to dict if "json" arg is provided.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_info(heketi_client_node, heketi_server_url, device_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s device info %s %s %s %s" % (
+        heketi_server_url, device_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    if json_arg:
+        device_info = json.loads(out)
+        return device_info
+    else:
+        return out
+
+
+def heketi_device_remove(heketi_client_node, heketi_server_url, device_id,
+                         raw_cli_output=False, **kwargs):
+    """Executes heketi device remove command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        device_id (str): Device id to remove device
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi device remove command output on success.
+        Tuple (ret, out, err): if raw_cli_output is True
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_device_remove(heketi_client_node, heketi_server_url, device_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s device remove %s %s %s %s" % (
+        heketi_server_url, device_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if raw_cli_output:
+        return ret, out, err
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    return out
+
+
+def heketi_node_delete(heketi_client_node, heketi_server_url, node_id,
+                       **kwargs):
+    """Executes heketi node delete command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url.
+        node_id (str): Node id to delete
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi node delete command output on success.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_node_delete(heketi_client_node, heketi_server_url, node_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s node delete %s %s %s %s" % (
+        heketi_server_url, node_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_node_disable(heketi_client_node, heketi_server_url, node_id,
+                        **kwargs):
+    """Executes heketi node disable command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        node_id (str): Node id to disable node
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi node disable command output on success.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_node_disable(heketi_client_node, heketi_server_url, node_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s node disable %s %s %s %s" % (
+        heketi_server_url, node_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_node_enable(heketi_client_node, heketi_server_url, node_id,
+                       **kwargs):
+    """Executes heketi node enable command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        node_id (str): Node id to enable device
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: heketi node enable command output on success.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_node_enable(heketi_client_node, heketi_server_url, node_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s node enable %s %s %s %s" % (
+        heketi_server_url, node_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    return out
+
+
+def heketi_node_info(heketi_client_node, heketi_server_url, node_id, **kwargs):
+    """Executes heketi node info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        node_id (str): Node ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: node info on success,
+        str: raw output if 'json' arg is not provided.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_node_info(heketi_client_node, heketi_server_url, node_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s node info %s %s %s %s" % (
+        heketi_server_url, node_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_node_list(heketi_client_node, heketi_server_url,
+                     heketi_user=None, heketi_secret=None):
+    """Execute CLI 'heketi node list' command and parse its output.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed
+        heketi_server_url (str): Heketi server url to perform request to
+        heketi_user (str): Name of the user to perform request with
+        heketi_secret (str): Secret for 'heketi_user'
+    Returns:
+        list of strings which are node IDs
+    Raises: openshiftstoragelibs.exceptions.ExecutionError when command fails.
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, user=heketi_user, secret=heketi_secret)
+
+    cmd = "heketi-cli -s %s node list %s %s %s" % (
+        heketi_server_url, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    heketi_node_id_list = []
+    for line in out.strip().split("\n"):
+        # Line looks like this: 'Id:nodeIdString\tCluster:clusterIdString'
+        heketi_node_id_list.append(
+            line.strip().split("Cluster")[0].strip().split(":")[1])
+    return heketi_node_id_list
+
+
+def heketi_blockvolume_info(heketi_client_node, heketi_server_url,
+                            block_volume_id, **kwargs):
+    """Executes heketi blockvolume info command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        block_volume_id (str): block volume ID
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: block volume info on success.
+        str: raw output if 'json' arg is not provided.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_blockvolume_info(heketi_client_node, block_volume_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s blockvolume info %s %s %s %s" % (
+        heketi_server_url, block_volume_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_blockvolume_create(heketi_client_node, heketi_server_url, size,
+                              **kwargs):
+    """Executes heketi blockvolume create
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        size (int): blockvolume size
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - name : (str)|None
+            - cluster : (str)|None
+            - ha : (int)|None
+            - auth : (bool)
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: blockvolume create info on success, only cli option is specified
+            without --json option, then it returns raw string output.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_blockvolume_create(heketi_client_node, heketi_server_url, size)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    auth = clusters = ha = name = None
+    if heketi_server_url is None:
+        heketi_server_url = ("http://" +
+                             "heketi-storage-project.cloudapps.mystorage.com")
+
+    if 'auth' in kwargs:
+        auth = kwargs['auth']
+    if 'clusters' in kwargs:
+        clusters = kwargs['clusters']
+    if 'ha' in kwargs:
+        ha = int(kwargs['ha'])
+    if 'name' in kwargs:
+        name = kwargs['name']
+
+    auth_arg = clusters_arg = ha_arg = name_arg = ''
+
+    if auth:
+        auth_arg = "--auth"
+    if clusters is not None:
+        clusters_arg = "--clusters %s" % clusters
+    if ha is not None:
+        ha_arg = "--ha %d" % ha
+    if name is not None:
+        name_arg = "--name %s" % name
+
+    cmd = ("heketi-cli -s %s blockvolume create --size=%s %s %s %s %s "
+           "%s %s %s %s" % (heketi_server_url, str(size), auth_arg,
+                            clusters_arg, ha_arg, name_arg, name_arg,
+                            admin_key, user, json_arg))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def heketi_blockvolume_delete(heketi_client_node, heketi_server_url,
+                              block_volume_id, raise_on_error=True, **kwargs):
+    """Executes heketi blockvolume delete command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+        block_volume_id (str): block volume ID
+        raise_on_error (bool): whether or not to raise exception
+            in case of an error.
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        str: volume delete command output on success
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_blockvolume_delete(heketi_client_node, heketi_server_url,
+                                  block_volume_id)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    err_msg = "Failed to delete '%s' volume. " % block_volume_id
+
+    cmd = "heketi-cli -s %s blockvolume delete %s %s %s %s" % (
+        heketi_server_url, block_volume_id, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        err_msg += "Out: %s, \nErr: %s" % (out, err)
+        g.log.error(err_msg)
+        if raise_on_error:
+            raise exceptions.ExecutionError(err_msg)
+    return out
+
+
+def heketi_blockvolume_list(heketi_client_node, heketi_server_url, **kwargs):
+    """Executes heketi blockvolume list command.
+
+    Args:
+        heketi_client_node (str): Node on which cmd has to be executed.
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        dict: volume list with --json on success, if cli option is specified
+            without --json option or with url, it returns raw string output.
+        False otherwise
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Example:
+        heketi_volume_info(heketi_client_node, heketi_server_url)
+    """
+
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = "heketi-cli -s %s blockvolume list %s %s %s" % (
+        heketi_server_url, json_arg, admin_key, user)
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, heketi_client_node, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if json_arg:
+        return json.loads(out)
+    return out
+
+
+def verify_volume_name_prefix(hostname, prefix, namespace, pvc_name,
+                              heketi_server_url, **kwargs):
+    """Checks whether heketi voluem is present with volname prefix or not.
+
+    Args:
+        hostname (str): hostname on which we want
+                        to check the heketi vol
+        prefix (str): volnameprefix given in storageclass
+        namespace (str): namespace
+        pvc_name (str): name of the pvc
+        heketi_server_url (str): Heketi server url
+
+    Kwargs:
+        The keys, values in kwargs are:
+            - json : (bool)
+            - secret : (str)|None
+            - user : (str)|None
+
+    Returns:
+        bool: True if volume found.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+    """
+    heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    heketi_vol_name_prefix = "%s_%s_%s_" % (prefix, namespace,  pvc_name)
+    cmd = "heketi-cli -s %s volume list %s %s %s | grep %s" % (
+        heketi_server_url, json_arg, admin_key, user, heketi_vol_name_prefix)
+    ret, out, err = g.run(hostname, cmd, "root")
+
+    if ret != 0:
+        msg = (
+            "Failed to execute '%s' command on '%s' node with following "
+            "error: %s" % (cmd, hostname, err))
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    output = out.strip()
+    g.log.info("heketi volume with volnameprefix present %s" % output)
+    return True
+
+
+def set_tags(heketi_client_node, heketi_server_url, source, source_id, tag,
+             **kwargs):
+    """Set any tags on Heketi node or device.
+
+    Args:
+        - heketi_client_node (str) : Node where we want to run our commands.
+            eg. "10.70.47.64"
+        - heketi_server_url (str) : This is a heketi server url
+            eg. "http://172.30.147.142:8080
+        - source (str) : This var is for node or device whether we
+                         want to set tag on node or device.
+                         Allowed values are "node" and "device".
+        - sorrce_id (str) : ID of node or device.
+            eg. "4f9c0249834919dd372e8fb3344cd7bd"
+        - tag (str) : This is a tag which we want to set
+            eg. "arbiter:required"
+    Kwargs:
+        user (str) : username
+        secret (str) : secret for that user
+    Returns:
+        True : if successful
+    Raises:
+        ValueError : when improper input data are provided.
+        exceptions.ExecutionError : when command fails.
+    """
+
+    if source not in ('node', 'device'):
+        msg = ("Incorrect value we can use 'node' or 'device' instead of %s."
+               % source)
+        g.log.error(msg)
+        raise ValueError(msg)
+
+    heketi_server_url, json_args, secret, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+
+    cmd = ("heketi-cli -s %s %s settags %s %s %s %s" %
+           (heketi_server_url, source, source_id, tag, user, secret))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if not ret:
+        g.log.info("Tagging of %s to %s is successful" % (source, tag))
+        return True
+
+    g.log.error(err)
+    raise exceptions.ExecutionError(err)
+
+
+def set_arbiter_tag(heketi_client_node, heketi_server_url, source,
+                    source_id, arbiter_tag_value, **kwargs):
+    """Set Arbiter tags on Heketi node or device.
+
+    Args:
+        - heketi_client_node (str) : node where we want to run our commands.
+            eg. "10.70.47.64"
+        - heketi_server_url (str) : This is a heketi server url
+            eg. "http://172.30.147.142:8080
+        - source (str) : This var is for node or device whether we
+                         want to set tag on node or device.
+                         Allowed values are "node" and "device".
+        - source_id (str) : ID of Heketi node or device
+            eg. "4f9c0249834919dd372e8fb3344cd7bd"
+        - arbiter_tag_value (str) : This is a tag which we want to set
+            Allowed values are "required", "disabled" and "supported".
+    Kwargs:
+        user (str) : username
+        secret (str) : secret for that user
+    Returns:
+        True : if successful
+    Raises:
+        ValueError : when improper input data are provided.
+        exceptions.ExecutionError : when command fails.
+    """
+
+    version = heketi_version.get_heketi_version(heketi_client_node)
+    if version < '6.0.0-11':
+        msg = ("heketi-client package %s does not support arbiter "
+               "functionality" % version.v_str)
+        g.log.error(msg)
+        raise NotImplementedError(msg)
+
+    if arbiter_tag_value in ('required', 'disabled', 'supported'):
+        arbiter_tag_value = "arbiter:%s" % arbiter_tag_value
+        return set_tags(heketi_client_node, heketi_server_url,
+                        source, source_id, arbiter_tag_value, **kwargs)
+
+    msg = ("Incorrect value we can use 'required', 'disabled', 'supported'"
+           "instead of %s" % arbiter_tag_value)
+    g.log.error(msg)
+    raise ValueError(msg)
+
+
+def rm_tags(heketi_client_node, heketi_server_url, source, source_id, tag,
+            **kwargs):
+    """Remove any kind of tags from Heketi node or device.
+
+    Args:
+        - heketi_client_node (str) : Node where we want to run our commands.
+            eg. "10.70.47.64"
+        - heketi_server_url (str) : This is a heketi server url
+            eg. "http://172.30.147.142:8080
+        - source (str) : This var is for node or device whether we
+                         want to set tag on node or device.
+                         Allowed values are "node" and "device".
+        - sorrce_id (str) : id of node or device
+            eg. "4f9c0249834919dd372e8fb3344cd7bd"
+        - tag (str) : This is a tag which we want to remove.
+    Kwargs:
+        user (str) : username
+        secret (str) : secret for that user
+    Returns:
+        True : if successful
+    Raises:
+        ValueError : when improper input data are provided.
+        exceptions.ExecutionError : when command fails.
+    """
+
+    heketi_server_url, json_args, secret, user = _set_heketi_global_flags(
+        heketi_server_url, **kwargs)
+    if source not in ('node', 'device'):
+        msg = ("Incorrect value we can use 'node' or 'device' instead of %s."
+               % source)
+        g.log.error(msg)
+        raise ValueError(msg)
+
+    cmd = ("heketi-cli -s %s %s rmtags %s %s %s %s" %
+           (heketi_server_url, source, source_id, tag, user, secret))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if not ret:
+        g.log.info("Removal of %s tag from %s is successful." % (tag, source))
+        return True
+
+    g.log.error(err)
+    raise exceptions.ExecutionError(err)
+
+
+def rm_arbiter_tag(heketi_client_node, heketi_server_url, source, source_id,
+                   **kwargs):
+    """Remove Arbiter tag from Heketi node or device.
+
+    Args:
+        - heketi_client_node (str) : Node where we want to run our commands.
+            eg. "10.70.47.64"
+        - heketi_server_url (str) : This is a heketi server url
+            eg. "http://172.30.147.142:8080
+        - source (str) : This var is for node or device whether we
+                         want to set tag on node or device.
+                         Allowed values are "node" and "device".
+        - source_id (str) : ID of Heketi node or device.
+            eg. "4f9c0249834919dd372e8fb3344cd7bd"
+    Kwargs:
+        user (str) : username
+        secret (str) : secret for that user
+    Returns:
+        True : if successful
+    Raises:
+        ValueError : when improper input data are provided.
+        exceptions.ExecutionError : when command fails.
+    """
+
+    version = heketi_version.get_heketi_version(heketi_client_node)
+    if version < '6.0.0-11':
+        msg = ("heketi-client package %s does not support arbiter "
+               "functionality" % version.v_str)
+        g.log.error(msg)
+        raise NotImplementedError(msg)
+
+    return rm_tags(heketi_client_node, heketi_server_url,
+                   source, source_id, 'arbiter', **kwargs)
+
+
+def get_heketi_metrics(heketi_client_node, heketi_server_url,
+                       prometheus_format=False):
+    """Execute curl command to get metrics output.
+
+    Args:
+        - heketi_client_node (str) : Node where we want to run our commands.
+        - heketi_server_url (str) : This is a heketi server url.
+        - prometheus_format (bool) : control the format of output
+            by default it is False, So it will parse prometheus format into
+            python dict. If we need prometheus format we have to set it True.
+
+    Raises:
+        exceptions.ExecutionError: if command fails.
+
+    Returns:
+        Metrics output: if successful
+    """
+
+    version = heketi_version.get_heketi_version(heketi_client_node)
+    if version < '6.0.0-14':
+        msg = ("heketi-client package %s does not support heketi "
+               "metrics functionality" % version.v_str)
+        g.log.error(msg)
+        raise NotImplementedError(msg)
+
+    cmd = "curl --max-time 10 %s/metrics" % heketi_server_url
+    ret, out, err = g.run(heketi_client_node, cmd)
+    if ret != 0:
+        msg = "failed to get Heketi metrics with following error: %s" % err
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+    if prometheus_format:
+        return out.strip()
+    return parse_prometheus_data(out)
+
+
+def heketi_examine_gluster(heketi_client_node, heketi_server_url):
+    """Execute heketi command to examine output from gluster servers.
+
+    Args:
+        - heketi_client_node (str): Node where we want to run our commands.
+        - heketi_server_url (str): This is a heketi server url.
+
+    Raises:
+        NotImplementedError: if heketi version is not expected
+        exceptions.ExecutionError: if command fails.
+
+    Returns:
+        dictionary: if successful
+    """
+
+    version = heketi_version.get_heketi_version(heketi_client_node)
+    if version < '8.0.0-7':
+        msg = ("heketi-client package %s does not support server state examine"
+               " gluster" % version.v_str)
+        g.log.error(msg)
+        raise NotImplementedError(msg)
+
+    heketi_server_url, json_arg, secret, user = _set_heketi_global_flags(
+        heketi_server_url)
+    # output is always json-like and we do not need to provide "--json" CLI arg
+    cmd = ("heketi-cli server state examine gluster -s %s %s %s"
+           % (heketi_server_url, user, secret))
+    ret, out, err = g.run(heketi_client_node, cmd)
+
+    if ret != 0:
+        msg = "failed to examine gluster with following error: %s" % err
+        g.log.error(msg)
+        raise exceptions.ExecutionError(msg)
+
+    return json.loads(out)
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_version.py b/openshift-storage-libs/openshiftstoragelibs/heketi_version.py
new file mode 100644
index 00000000..0da81176
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/heketi_version.py
@@ -0,0 +1,246 @@
+"""
+Use this module for any Heketi server and client packages versions comparisons.
+
+Usage example:
+
+    # Assume Heketi server version is '7.0.0-3' and client is '7.0.0-5'
+    Then we have following:
+
+    from openshiftstoragelibs import heketi_version
+    version = heketi_version.get_heketi_version()
+    if version < '7.0.0-4':
+        # True
+    if version < '7.0.0-2':
+        # False
+    if '7.0.0-2' < version <= '7.0.0-3':
+        # True
+
+    At first step, we compare requested version against the Heketi server
+    version, making sure they are compatible. Then, we make sure, that
+    existing heketi client package has either the same or newer version than
+    server's one.
+"""
+import re
+
+from glusto.core import Glusto as g
+import six
+
+from openshiftstoragelibs import command
+from openshiftstoragelibs import exceptions
+
+
+HEKETI_VERSION_RE = r"(\d+)(?:\.)(\d+)(?:\.)(\d+)(?:\-)(\d+)$"
+HEKETI_CLIENT_VERSION = None
+HEKETI_SERVER_VERSION = None
+
+
+def _get_heketi_client_version_str(hostname=None):
+    """Gets Heketi client package version from heketi client node.
+
+    Args:
+        hostname (str): Node on which the version check command should run.
+    Returns:
+        str : heketi version, i.e. '7.0.0-1'
+    Raises: 'exceptions.ExecutionError' if failed to get version
+    """
+    if not hostname:
+        openshift_config = g.config.get("cns", g.config.get("openshift"))
+        heketi_config = openshift_config['heketi_config']
+        hostname = heketi_config['heketi_client_node'].strip()
+    cmd = ("rpm -q heketi-client --queryformat '%{version}-%{release}\n' | "
+           "cut -d '.' -f 1,2,3")
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret != 0:
+        msg = ("Failed to get heketi client version. "
+               "\n'err': %s\n 'out': %s" % (err, out))
+        g.log.error(msg)
+        raise AssertionError(msg)
+    out = out.strip()
+    if not out:
+        error_msg = "Empty output for '%s' cmd: '%s'" % (cmd, out)
+        g.log.error(error_msg)
+        raise exceptions.ExecutionError(error_msg)
+
+    return out
+
+
+def _get_heketi_server_version_str(ocp_client_node=None):
+    """Gets Heketi server package version from Heketi POD.
+
+    Args:
+        ocp_client_node (str): Node on which the version check command should
+                               run.
+    Returns:
+        str : heketi version, i.e. '7.0.0-1'
+    Raises: 'exceptions.ExecutionError' if failed to get version
+    """
+    if not ocp_client_node:
+        ocp_client_node = g.config["ocp_servers"]["client"].keys()[0]
+    get_package_version_cmd = (
+        "rpm -q heketi --queryformat '%{version}-%{release}\n' | "
+        "cut -d '.' -f 1,2,3")
+
+    # NOTE(vponomar): we implement Heketi POD call command here, not in common
+    # module for OC commands just to avoid cross-reference imports.
+    get_pods_cmd = "oc get -o wide --no-headers=true pods --selector heketi"
+    heketi_pods = command.cmd_run(get_pods_cmd, hostname=ocp_client_node)
+
+    err_msg = ""
+    for heketi_pod_line in heketi_pods.split("\n"):
+        heketi_pod_data = heketi_pod_line.split()
+        if ("-deploy" in heketi_pod_data[0] or
+                heketi_pod_data[1].lower() != "1/1" or
+                heketi_pod_data[2].lower() != "running"):
+            continue
+        try:
+            pod_cmd = "oc exec %s -- %s" % (
+                heketi_pod_data[0], get_package_version_cmd)
+            return command.cmd_run(pod_cmd, hostname=ocp_client_node)
+        except Exception as e:
+            err = ("Failed to run '%s' command on '%s' Heketi POD. "
+                   "Error: %s\n" % (pod_cmd, heketi_pod_data[0], e))
+            err_msg += err
+            g.log.error(err)
+    if not err_msg:
+        err_msg += "Haven't found 'Running' and 'ready' (1/1) Heketi PODs.\n"
+    err_msg += "Heketi PODs: %s" % heketi_pods
+    raise exceptions.ExecutionError(err_msg)
+
+
+def _parse_heketi_version(heketi_version_str):
+    """Parses Heketi version str into tuple of 4 values.
+
+    Args:
+        heketi_version_str (str): Heketi version like '7.0.0-1'
+    Returns:
+        Tuple object of 4 values - major, minor, micro and build version parts.
+    """
+    groups = re.findall(HEKETI_VERSION_RE, heketi_version_str)
+    err_msg = (
+        "Failed to parse '%s' str into 4 Heketi version parts. "
+        "Expected value like '7.0.0-1'" % heketi_version_str)
+    assert groups, err_msg
+    assert len(groups) == 1, err_msg
+    assert len(groups[0]) == 4, err_msg
+    return (int(groups[0][0]), int(groups[0][1]),
+            int(groups[0][2]), int(groups[0][3]))
+
+
+class HeketiVersion(object):
+    """Eases Heketi versions comparison.
+
+    Instance of this class can be used for comparison with other instance of
+    it or to string-like objects.
+
+    Input str version is required to have 4 version parts -
+    'major', 'minor', 'micro' and 'build' versions. Example - '7.0.0-1'
+
+    Usage example (1) - compare to string object:
+        version_7_0_0_2 = HeketiVersion('7.0.0-2')
+        cmp_result = '7.0.0-1' < version_7_0_0_2 <= '8.0.0-1'
+
+    Usage example (2) - compare to the same type of an object:
+        version_7_0_0_1 = HeketiVersion('7.0.0-1')
+        version_7_0_0_2 = HeketiVersion('7.0.0-2')
+        cmp_result = version_7_0_0_1 < version_7_0_0_2
+    """
+    def __init__(self, heketi_version_str):
+        self.v = _parse_heketi_version(heketi_version_str)
+        self.v_str = heketi_version_str
+        self.major, self.minor, self.micro, self.build = self.v
+
+    def __str__(self):
+        return self.v_str
+
+    def _adapt_other(self, other):
+        if isinstance(other, six.string_types):
+            return HeketiVersion(other)
+        elif isinstance(other, HeketiVersion):
+            return other
+        else:
+            raise NotImplementedError(
+                "'%s' type is not supported for Heketi version "
+                "comparison." % type(other))
+
+    def _compare_client_and_server_versions(self, client_v, server_v):
+        if client_v < server_v:
+            raise Exception(
+                "Client version (%s) is older than server's (%s)." % (
+                    client_v, server_v))
+
+    def __lt__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v < adapted_other.v
+
+    def __le__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v <= adapted_other.v
+
+    def __eq__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v == adapted_other.v
+
+    def __ge__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v >= adapted_other.v
+
+    def __gt__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v > adapted_other.v
+
+    def __ne__(self, other):
+        global HEKETI_CLIENT_VERSION
+        global HEKETI_SERVER_VERSION
+        self._compare_client_and_server_versions(
+            HEKETI_CLIENT_VERSION.v, HEKETI_SERVER_VERSION.v)
+        adapted_other = self._adapt_other(other)
+        return self.v != adapted_other.v
+
+
+def get_heketi_version(hostname=None, ocp_client_node=None):
+    """Cacher of the Heketi client package version.
+
+    Version of Heketi client package is constant value. So, we call API just
+    once and then reuse it's output.
+
+    Args:
+        hostname (str): a node with 'heketi' client where command should run on
+            If not specified, then first key
+            from 'openshift.heketi_config.heketi_client_node' config option
+            will be picked up.
+        ocp_client_node (str): a node with the 'oc' client,
+            where Heketi POD command will run.
+            If not specified, then first key
+            from 'ocp_servers.client' config option will be picked up.
+    Returns:
+        HeketiVersion object instance.
+    """
+    global HEKETI_CLIENT_VERSION
+    global HEKETI_SERVER_VERSION
+    if not (HEKETI_SERVER_VERSION and HEKETI_CLIENT_VERSION):
+        client_version_str = _get_heketi_client_version_str(hostname=hostname)
+        server_version_str = _get_heketi_server_version_str(
+            ocp_client_node=ocp_client_node)
+        HEKETI_CLIENT_VERSION = HeketiVersion(client_version_str)
+        HEKETI_SERVER_VERSION = HeketiVersion(server_version_str)
+    return HEKETI_SERVER_VERSION
diff --git a/openshift-storage-libs/openshiftstoragelibs/naming.py b/openshift-storage-libs/openshiftstoragelibs/naming.py
new file mode 100644
index 00000000..b44559ad
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/naming.py
@@ -0,0 +1,56 @@
+"""Helper functions for working with names for volumes, resources, etc.
+"""
+
+import string
+import random
+import re
+
+# we only use lowercase here because kubernetes requires
+# names to be lowercase or digits, so that is our default
+UNIQUE_CHARS = (string.lowercase + string.digits)
+
+
+def make_unique_label(prefix=None, suffix=None, sep='-',
+                      clean=r'[^a-zA-Z0-9]+', unique_len=8,
+                      unique_chars=UNIQUE_CHARS):
+    """Generate a unique name string based on an optional prefix,
+    suffix, and pseudo-random set of alphanumeric characters.
+
+    Args:
+        prefix (str): Start of the unique string.
+        suffix (str): End of the unique string.
+        sep (str): Separator string (between sections/invalid chars).
+        clean (str): Reqular expression matching invalid chars.
+            that will be replaced by `sep` if found in the prefix or suffix
+        unique_len (int): Length of the unique part.
+        unique_chars (str): String representing the set of characters
+            the unique part will draw from.
+    Returns:
+        str: The uniqueish string.
+    """
+    cre = re.compile(clean)
+    parts = []
+    if prefix:
+        parts.append(cre.sub(sep, prefix))
+    parts.append(''.join(random.choice(unique_chars)
+                         for _ in range(unique_len)))
+    if suffix:
+        parts.append(cre.sub(sep, suffix))
+    return sep.join(parts)
+
+
+def extract_method_name(full_name, keep_class=False):
+    """Given a full test name as returned from TestCase.id() return
+    just the method part or class.method.
+
+    Args:
+        full_name (str): Dot separated name of test.
+        keep_class (str): Retain the class name, if false only the
+            method name will be returned.
+    Returns:
+        str: Method name or class.method_name.
+    """
+    offset = -1
+    if keep_class:
+        offset = -2
+    return '.'.join(full_name.split('.')[offset:])
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
new file mode 100644
index 00000000..295dc42b
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
@@ -0,0 +1,1507 @@
+"""Library for openshift operations.
+
+Various utility functions for interacting with OCP/OpenShift.
+"""
+
+import base64
+import json
+import re
+import types
+
+from glusto.core import Glusto as g
+from glustolibs.gluster import volume_ops
+import mock
+import yaml
+
+from openshiftstoragelibs import command
+from openshiftstoragelibs import exceptions
+from openshiftstoragelibs import openshift_version
+from openshiftstoragelibs import utils
+from openshiftstoragelibs import waiter
+from openshiftstoragelibs.heketi_ops import (
+    heketi_blockvolume_info,
+    heketi_volume_info,
+)
+
+PODS_WIDE_RE = re.compile(
+    r'(\S+)\s+(\S+)\s+(\w+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+).*\n')
+SERVICE_STATUS = "systemctl status %s"
+SERVICE_RESTART = "systemctl restart %s"
+SERVICE_STATUS_REGEX = r"Active: active \((.*)\) since .*;.*"
+
+
+def oc_get_pods(ocp_node, selector=None):
+    """Gets the pods info with 'wide' option in the current project.
+
+    Args:
+        ocp_node (str): Node in which ocp command will be executed.
+        selector (str): optional option. Selector for OCP pods.
+            example: "glusterfs-node=pod" for filtering out only Gluster PODs.
+
+    Returns:
+        dict : dict of pods info in the current project.
+    """
+
+    cmd = "oc get -o wide --no-headers=true pods"
+    if selector:
+        cmd += " --selector %s" % selector
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error("Failed to get ocp pods on node %s" % ocp_node)
+        raise AssertionError('failed to get pods: %r' % (err,))
+    return _parse_wide_pods_output(out)
+
+
+def _parse_wide_pods_output(output):
+    """Parse the output of `oc get -o wide pods`.
+    """
+    # Interestingly, the output of get pods is "cooked" in such a way that
+    # the values in the ready, status, & restart fields are not accessible
+    # from YAML/JSON/templating forcing us to scrape the output for
+    # these values
+    # (at the time of this writing the logic is in
+    #  printPodBase in kubernetes/pkg/printers/internalversion/printers.go )
+    # Possibly obvious, but if you don't need those values you can
+    # use the YAML output directly.
+    #
+    # TODO: Add unit tests for this parser
+    pods_info = {}
+    for each_pod_info in PODS_WIDE_RE.findall(output):
+        pods_info[each_pod_info[0]] = {
+            'ready': each_pod_info[1],
+            'status': each_pod_info[2],
+            'restarts': each_pod_info[3],
+            'age': each_pod_info[4],
+            'ip': each_pod_info[5],
+            'node': each_pod_info[6],
+        }
+    return pods_info
+
+
+def oc_get_pods_full(ocp_node):
+    """Gets all the pod info via YAML in the current project.
+
+    Args:
+        ocp_node (str): Node in which ocp command will be executed.
+
+    Returns:
+        dict: The YAML output converted to python objects
+            (a top-level dict)
+    """
+
+    cmd = "oc get -o yaml pods"
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error("Failed to get ocp pods on node %s" % ocp_node)
+        raise AssertionError('failed to get pods: %r' % (err,))
+    return yaml.load(out)
+
+
+def get_ocp_gluster_pod_names(ocp_node):
+    """Gets the gluster pod names in the current project.
+
+    Args:
+        ocp_node (str): Node in which ocp command will be executed.
+
+    Returns:
+        list : list of gluster pod names in the current project.
+            Empty list, if there are no gluster pods.
+
+    Example:
+        get_ocp_gluster_pod_names(ocp_node)
+    """
+
+    pod_names = oc_get_pods(ocp_node).keys()
+    return [pod for pod in pod_names if pod.startswith('glusterfs-')]
+
+
+def get_amount_of_gluster_nodes(ocp_node):
+    """Calculate amount of Gluster nodes.
+
+    Args:
+        ocp_node (str): node to run 'oc' commands on.
+    Returns:
+        Integer value as amount of either GLuster PODs or Gluster nodes.
+    """
+    # Containerized Gluster
+    gluster_pods = get_ocp_gluster_pod_names(ocp_node)
+    if gluster_pods:
+        return len(gluster_pods)
+
+    # Standalone Gluster
+    configured_gluster_nodes = len(g.config.get("gluster_servers", {}))
+    if configured_gluster_nodes:
+        return configured_gluster_nodes
+
+    raise exceptions.ConfigError(
+        "Haven't found neither Gluster PODs nor Gluster nodes.")
+
+
+def switch_oc_project(ocp_node, project_name):
+    """Switch to the given project.
+
+    Args:
+        ocp_node (str): Node in which ocp command will be executed.
+        project_name (str): Project name.
+    Returns:
+        bool : True on switching to given project.
+            False otherwise
+
+    Example:
+        switch_oc_project(ocp_node, "storage-project")
+    """
+
+    cmd = "oc project %s" % project_name
+    ret, _, _ = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error("Failed to switch to project %s" % project_name)
+        return False
+    return True
+
+
+def oc_rsync(ocp_node, pod_name, src_dir_path, dest_dir_path):
+    """Sync file from 'src_dir_path' path on ocp_node to
+       'dest_dir_path' path on 'pod_name' using 'oc rsync' command.
+
+    Args:
+        ocp_node (str): Node on which oc rsync command will be executed
+        pod_name (str): Name of the pod on which source directory to be
+                        mounted
+        src_dir_path (path): Source path from which directory to be mounted
+        dest_dir_path (path): destination path to which directory to be
+                              mounted
+    """
+    ret, out, err = g.run(ocp_node, ['oc',
+                                     'rsync',
+                                     src_dir_path,
+                                     '%s:%s' % (pod_name, dest_dir_path)])
+    if ret != 0:
+        error_msg = 'failed to sync directory in pod: %r; %r' % (out, err)
+        g.log.error(error_msg)
+        raise AssertionError(error_msg)
+
+
+def oc_rsh(ocp_node, pod_name, command, log_level=None):
+    """Run a command in the ocp pod using `oc rsh`.
+
+    Args:
+        ocp_node (str): Node on which oc rsh command will be executed.
+        pod_name (str): Name of the pod on which the command will
+            be executed.
+        command (str|list): command to run.
+        log_level (str|None): log level to be passed to glusto's run
+            method.
+
+    Returns:
+        A tuple consisting of the command return code, stdout, and stderr.
+    """
+    prefix = ['oc', 'rsh', pod_name]
+    if isinstance(command, types.StringTypes):
+        cmd = ' '.join(prefix + [command])
+    else:
+        cmd = prefix + command
+
+    # unpack the tuple to make sure our return value exactly matches
+    # our docstring
+    ret, stdout, stderr = g.run(ocp_node, cmd, log_level=log_level)
+    return (ret, stdout, stderr)
+
+
+def oc_create(ocp_node, value, value_type='file'):
+    """Create a resource based on the contents of the given file name.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run
+        value (str): Filename (on remote) or file data
+            to be passed to oc create command.
+        value_type (str): either 'file' or 'stdin'.
+    Raises:
+        AssertionError: Raised when resource fails to create.
+    """
+    if value_type == 'file':
+        cmd = ['oc', 'create', '-f', value]
+    else:
+        cmd = ['echo', '\'%s\'' % value, '|', 'oc', 'create', '-f', '-']
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        msg = 'Failed to create resource: %r; %r' % (out, err)
+        g.log.error(msg)
+        raise AssertionError(msg)
+    g.log.info('Created resource from %s.' % value_type)
+
+
+def oc_process(ocp_node, params, filename):
+    """Create a resource template based on the contents of the
+       given filename and params provided.
+    Args:
+        ocp_node (str): Node on which the ocp command will run
+        filename (str): Filename (on remote) to be passed to
+                        oc process command.
+    Returns: template generated through process command
+    Raises:
+        AssertionError: Raised when resource fails to create.
+    """
+
+    ret, out, err = g.run(ocp_node, ['oc', 'process', '-f', filename, params])
+    if ret != 0:
+        error_msg = 'failed to create process: %r; %r' % (out, err)
+        g.log.error(error_msg)
+        raise AssertionError(error_msg)
+    g.log.info('Created resource from file (%s)', filename)
+
+    return out
+
+
+def oc_create_secret(hostname, secret_name_prefix="autotests-secret-",
+                     namespace="default",
+                     data_key="password",
+                     secret_type="kubernetes.io/glusterfs"):
+    """Create secret using data provided as stdin input.
+
+    Args:
+        hostname (str): Node on which 'oc create' command will be executed.
+        secret_name_prefix (str): secret name will consist of this prefix and
+                                  random str.
+        namespace (str): name of a namespace to create a secret in
+        data_key (str): plain text value for secret which will be transformed
+                        into base64 string automatically.
+        secret_type (str): type of the secret, which will be created.
+    Returns: name of a secret
+    """
+    secret_name = "%s-%s" % (secret_name_prefix, utils.get_random_str())
+    secret_data = json.dumps({
+        "apiVersion": "v1",
+        "data": {"key": base64.b64encode(data_key)},
+        "kind": "Secret",
+        "metadata": {
+            "name": secret_name,
+            "namespace": namespace,
+        },
+        "type": secret_type,
+    })
+    oc_create(hostname, secret_data, 'stdin')
+    return secret_name
+
+
+def oc_create_sc(hostname, sc_name_prefix="autotests-sc",
+                 provisioner="kubernetes.io/glusterfs",
+                 allow_volume_expansion=False,
+                 reclaim_policy="Delete", **parameters):
+    """Create storage class using data provided as stdin input.
+
+    Args:
+        hostname (str): Node on which 'oc create' command will be executed.
+        sc_name_prefix (str): sc name will consist of this prefix and
+                              random str.
+        provisioner (str): name of the provisioner
+        allow_volume_expansion (bool): Set it to True if need to allow
+                                       volume expansion.
+    Kvargs:
+        All the keyword arguments are expected to be key and values of
+        'parameters' section for storage class.
+    """
+    allowed_parameters = (
+        'resturl', 'secretnamespace', 'restuser', 'secretname',
+        'restauthenabled', 'restsecretnamespace', 'restsecretname',
+        'hacount', 'clusterids', 'chapauthenabled', 'volumenameprefix',
+        'volumeoptions', 'volumetype'
+    )
+    for parameter in parameters.keys():
+        if parameter.lower() not in allowed_parameters:
+            parameters.pop(parameter)
+    sc_name = "%s-%s" % (sc_name_prefix, utils.get_random_str())
+    sc_data = json.dumps({
+        "kind": "StorageClass",
+        "apiVersion": "storage.k8s.io/v1",
+        "metadata": {"name": sc_name},
+        "provisioner": provisioner,
+        "reclaimPolicy": reclaim_policy,
+        "parameters": parameters,
+        "allowVolumeExpansion": allow_volume_expansion,
+    })
+    oc_create(hostname, sc_data, 'stdin')
+    return sc_name
+
+
+def oc_create_pvc(hostname, sc_name=None, pvc_name_prefix="autotests-pvc",
+                  pvc_size=1):
+    """Create PVC using data provided as stdin input.
+
+    Args:
+        hostname (str): Node on which 'oc create' command will be executed.
+        sc_name (str): name of a storage class to create PVC in.
+        pvc_name_prefix (str): PVC name will consist of this prefix and
+                               random str.
+        pvc_size (int/str): size of PVC in Gb
+    """
+    pvc_name = "%s-%s" % (pvc_name_prefix, utils.get_random_str())
+    metadata = {"name": pvc_name}
+    if sc_name:
+        metadata["annotations"] = {
+            "volume.kubernetes.io/storage-class": sc_name,
+            "volume.beta.kubernetes.io/storage-class": sc_name,
+        }
+    pvc_data = json.dumps({
+        "kind": "PersistentVolumeClaim",
+        "apiVersion": "v1",
+        "metadata": metadata,
+        "spec": {
+            "accessModes": ["ReadWriteOnce"],
+            "resources": {"requests": {"storage": "%sGi" % pvc_size}}
+        },
+    })
+    oc_create(hostname, pvc_data, 'stdin')
+    return pvc_name
+
+
+def oc_create_app_dc_with_io(
+        hostname, pvc_name, dc_name_prefix="autotests-dc-with-app-io",
+        replicas=1, space_to_use=1048576):
+    """Create DC with app PODs and attached PVC, constantly running I/O.
+
+    Args:
+        hostname (str): Node on which 'oc create' command will be executed.
+        pvc_name (str): name of the Persistent Volume Claim to attach to
+                        the application PODs where constant I/O will run.
+        dc_name_prefix (str): DC name will consist of this prefix and
+                              random str.
+        replicas (int): amount of application POD replicas.
+        space_to_use (int): value in bytes which will be used for I/O.
+    """
+    dc_name = "%s-%s" % (dc_name_prefix, utils.get_random_str())
+    container_data = {
+        "name": dc_name,
+        "image": "cirros",
+        "volumeMounts": [{"mountPath": "/mnt", "name": dc_name}],
+        "command": ["sh"],
+        "args": [
+            "-ec",
+            "trap \"rm -f /mnt/random-data-$HOSTNAME.log ; exit 0\" SIGTERM; "
+            "while true; do "
+            " (mount | grep '/mnt') && "
+            "  (head -c %s < /dev/urandom > /mnt/random-data-$HOSTNAME.log) ||"
+            "   exit 1; "
+            " sleep 1 ; "
+            "done" % space_to_use,
+        ],
+        "livenessProbe": {
+            "initialDelaySeconds": 3,
+            "periodSeconds": 3,
+            "exec": {"command": [
+                "sh", "-ec",
+                "mount | grep '/mnt' && "
+                "  head -c 1 < /dev/urandom >> /mnt/random-data-$HOSTNAME.log"
+            ]},
+        },
+    }
+    dc_data = json.dumps({
+        "kind": "DeploymentConfig",
+        "apiVersion": "v1",
+        "metadata": {"name": dc_name},
+        "spec": {
+            "replicas": replicas,
+            "triggers": [{"type": "ConfigChange"}],
+            "paused": False,
+            "revisionHistoryLimit": 2,
+            "template": {
+                "metadata": {"labels": {"name": dc_name}},
+                "spec": {
+                    "restartPolicy": "Always",
+                    "volumes": [{
+                        "name": dc_name,
+                        "persistentVolumeClaim": {"claimName": pvc_name},
+                    }],
+                    "containers": [container_data],
+                    "terminationGracePeriodSeconds": 20,
+                }
+            }
+        }
+    })
+    oc_create(hostname, dc_data, 'stdin')
+    return dc_name
+
+
+def oc_create_tiny_pod_with_volume(hostname, pvc_name, pod_name_prefix='',
+                                   mount_path='/mnt'):
+    """Create tiny POD from image in 10Mb with attached volume at /mnt"""
+    pod_name = "%s-%s" % (pod_name_prefix, utils.get_random_str())
+    pod_data = json.dumps({
+        "apiVersion": "v1",
+        "kind": "Pod",
+        "metadata": {
+            "name": pod_name,
+        },
+        "spec": {
+            "terminationGracePeriodSeconds": 20,
+            "containers": [{
+                "name": pod_name,
+                "image": "cirros",  # noqa: 10 Mb! linux image
+                "volumeMounts": [{"mountPath": mount_path, "name": "vol"}],
+                "command": [
+                    "/bin/sh", "-ec",
+                    "trap 'exit 0' SIGTERM ; "
+                    "while :; do echo '.'; sleep 5 ; done",
+                ]
+            }],
+            "volumes": [{
+                "name": "vol",
+                "persistentVolumeClaim": {"claimName": pvc_name},
+            }],
+            "restartPolicy": "Never",
+        }
+    })
+    oc_create(hostname, pod_data, 'stdin')
+    return pod_name
+
+
+def oc_delete(ocp_node, rtype, name, raise_on_absence=True):
+    """Delete an OCP resource by name.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        rtype (str): Name of the resource type (pod, storageClass, etc).
+        name (str): Name of the resource to delete.
+        raise_on_absence (bool): if resource absent raise
+                                 exception if value is true,
+                                 else return
+                                 default value: True
+    """
+    if not oc_get_yaml(ocp_node, rtype, name,
+                       raise_on_error=raise_on_absence):
+        return
+    cmd = ['oc', 'delete', rtype, name]
+    if openshift_version.get_openshift_version() >= '3.11':
+        cmd.append('--wait=false')
+
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error('Failed to delete resource: %s, %s: %r; %r',
+                    rtype, name, out, err)
+        raise AssertionError('failed to delete resource: %r; %r' % (out, err))
+    g.log.info('Deleted resource: %r %r', rtype, name)
+
+
+def oc_get_custom_resource(ocp_node, rtype, custom, name=None, selector=None,
+                           raise_on_error=True):
+    """Get an OCP resource by custom column names.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        rtype (str): Name of the resource type (pod, storageClass, etc).
+        custom (str): Name of the custom columm to fetch.
+        name (str|None): Name of the resource to fetch.
+        selector (str|list|None): Column Name or list of column
+                                  names select to.
+        raise_on_error (bool): If set to true a failure to fetch
+            resource inforation will raise an error, otherwise
+            an empty dict will be returned.
+    Returns:
+        list: List containting data about the resource custom column
+    Raises:
+        AssertionError: Raised when unable to get resource and
+            `raise_on_error` is true.
+    Example:
+        Get all "pvc" with "metadata.name" parameter values:
+            pvc_details = oc_get_custom_resource(
+                ocp_node, "pvc", ":.metadata.name"
+            )
+    """
+    cmd = ['oc', 'get', rtype, '--no-headers']
+
+    cmd.append('-o=custom-columns=%s' % (
+        ','.join(custom) if isinstance(custom, list) else custom))
+
+    if selector:
+        cmd.append('--selector %s' % (
+            ','.join(selector) if isinstance(selector, list) else selector))
+
+    if name:
+        cmd.append(name)
+
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error('Failed to get %s: %s: %r', rtype, name, err)
+        if raise_on_error:
+            raise AssertionError('failed to get %s: %s: %r'
+                                 % (rtype, name, err))
+        return []
+
+    if name:
+        return filter(None, map(str.strip, (out.strip()).split(' ')))
+    else:
+        out_list = []
+        for line in (out.strip()).split('\n'):
+            out_list.append(filter(None, map(str.strip, line.split(' '))))
+        return out_list
+
+
+def oc_get_yaml(ocp_node, rtype, name=None, raise_on_error=True):
+    """Get an OCP resource by name.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        rtype (str): Name of the resource type (pod, storageClass, etc).
+        name (str|None): Name of the resource to fetch.
+        raise_on_error (bool): If set to true a failure to fetch
+            resource inforation will raise an error, otherwise
+            an empty dict will be returned.
+    Returns:
+        dict: Dictionary containting data about the resource
+    Raises:
+        AssertionError: Raised when unable to get resource and
+            `raise_on_error` is true.
+    """
+    cmd = ['oc', 'get', '-oyaml', rtype]
+    if name is not None:
+        cmd.append(name)
+    ret, out, err = g.run(ocp_node, cmd)
+    if ret != 0:
+        g.log.error('Failed to get %s: %s: %r', rtype, name, err)
+        if raise_on_error:
+            raise AssertionError('failed to get %s: %s: %r'
+                                 % (rtype, name, err))
+        return {}
+    return yaml.load(out)
+
+
+def oc_get_pvc(ocp_node, name):
+    """Get information on a persistant volume claim.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        name (str): Name of the PVC.
+    Returns:
+        dict: Dictionary containting data about the PVC.
+    """
+    return oc_get_yaml(ocp_node, 'pvc', name)
+
+
+def oc_get_pv(ocp_node, name):
+    """Get information on a persistant volume.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+        name (str): Name of the PV.
+    Returns:
+        dict: Dictionary containting data about the PV.
+    """
+    return oc_get_yaml(ocp_node, 'pv', name)
+
+
+def oc_get_all_pvs(ocp_node):
+    """Get information on all persistent volumes.
+
+    Args:
+        ocp_node (str): Node on which the ocp command will run.
+    Returns:
+        dict: Dictionary containting data about the PV.
+    """
+    return oc_get_yaml(ocp_node, 'pv', None)
+
+
+def create_namespace(hostname, namespace):
+    '''
+     This function creates namespace
+     Args:
+         hostname (str): hostname on which we need to
+                         create namespace
+         namespace (str): project name
+     Returns:
+         bool: True if successful and if already exists,
+               otherwise False
+    '''
+    cmd = "oc new-project %s" % namespace
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret == 0:
+        g.log.info("new namespace %s successfully created" % namespace)
+        return True
+    output = out.strip().split("\n")[0]
+    if "already exists" in output:
+        g.log.info("namespace %s already exists" % namespace)
+        return True
+    g.log.error("failed to create namespace %s" % namespace)
+    return False
+
+
+def wait_for_resource_absence(ocp_node, rtype, name,
+                              interval=5, timeout=300):
+    _waiter = waiter.Waiter(timeout=timeout, interval=interval)
+    resource, pv_name = None, None
+    for w in _waiter:
+        try:
+            resource = oc_get_yaml(ocp_node, rtype, name, raise_on_error=True)
+        except AssertionError:
+            break
+    if rtype == 'pvc':
+        cmd = "oc get pv -o=custom-columns=:.spec.claimRef.name | grep %s" % (
+            name)
+        for w in _waiter:
+            ret, out, err = g.run(ocp_node, cmd, "root")
+            _pv_name = out.strip()
+            if _pv_name and not pv_name:
+                pv_name = _pv_name
+            if ret != 0:
+                break
+    if w.expired:
+        # Gather more info for ease of debugging
+        try:
+            r_events = get_events(ocp_node, obj_name=name)
+        except Exception:
+            r_events = '?'
+        error_msg = (
+            "%s '%s' still exists after waiting for it %d seconds.\n"
+            "Resource info: %s\n"
+            "Resource related events: %s" % (
+                rtype, name, timeout, resource, r_events))
+        if rtype == 'pvc' and pv_name:
+            try:
+                pv_events = get_events(ocp_node, obj_name=pv_name)
+            except Exception:
+                pv_events = '?'
+            error_msg += "\nPV events: %s" % pv_events
+
+        g.log.error(error_msg)
+        raise exceptions.ExecutionError(error_msg)
+
+
+def scale_dc_pod_amount_and_wait(hostname, dc_name,
+                                 pod_amount=1, namespace=None):
+    """Scale amount of PODs for a DC.
+
+    If pod_amount is 0, then wait for it's absence.
+    If pod_amount => 1, then wait for all of a DC PODs to be ready.
+
+    Args:
+        hostname (str): Node on which the ocp command will run
+        dc_name (str): Name of heketi dc
+        pod_amount (int): Number of PODs to scale. Default is 1.
+        namespace (str): Namespace of a DC.
+    """
+    namespace_arg = "--namespace=%s" % namespace if namespace else ""
+    scale_cmd = "oc scale --replicas=%d dc/%s %s" % (
+        pod_amount, dc_name, namespace_arg)
+    command.cmd_run(scale_cmd, hostname=hostname)
+
+    pod_names = get_pod_names_from_dc(hostname, dc_name)
+    for pod_name in pod_names:
+        if pod_amount == 0:
+            wait_for_resource_absence(hostname, 'pod', pod_name)
+        else:
+            wait_for_pod_be_ready(hostname, pod_name)
+    return pod_names
+
+
+def get_gluster_pod_names_by_pvc_name(ocp_node, pvc_name):
+    """Get Gluster POD names, whose nodes store bricks for specified PVC.
+
+    Args:
+        ocp_node (str): Node to execute OCP commands on.
+        pvc_name (str): Name of a PVC to get related Gluster PODs.
+    Returns:
+        list: List of dicts, which consist of following 3 key-value pairs:
+            pod_name=<pod_name_value>,
+            host_name=<host_name_value>,
+            host_ip=<host_ip_value>
+    """
+    # Check storage provisioner
+    sp_cmd = (
+        r'oc get pvc %s --no-headers -o=custom-columns='
+        r':.metadata.annotations."volume\.beta\.kubernetes\.io\/'
+        r'storage\-provisioner"' % pvc_name)
+    sp_raw = command.cmd_run(sp_cmd, hostname=ocp_node)
+    sp = sp_raw.strip()
+
+    # Get node IPs
+    if sp == "kubernetes.io/glusterfs":
+        pv_info = get_gluster_vol_info_by_pvc_name(ocp_node, pvc_name)
+        gluster_pod_nodes_ips = [
+            brick["name"].split(":")[0]
+            for brick in pv_info["bricks"]["brick"]
+        ]
+    elif sp == "gluster.org/glusterblock":
+        get_gluster_pod_node_ip_cmd = (
+            r"""oc get pv --template '{{range .items}}""" +
+            r"""{{if eq .spec.claimRef.name "%s"}}""" +
+            r"""{{.spec.iscsi.targetPortal}}{{" "}}""" +
+            r"""{{.spec.iscsi.portals}}{{end}}{{end}}'""") % (
+                pvc_name)
+        node_ips_raw = command.cmd_run(
+            get_gluster_pod_node_ip_cmd, hostname=ocp_node)
+        node_ips_raw = node_ips_raw.replace(
+            "[", " ").replace("]", " ").replace(",", " ")
+        gluster_pod_nodes_ips = [
+            s.strip() for s in node_ips_raw.split(" ") if s.strip()
+        ]
+    else:
+        assert False, "Unexpected storage provisioner: %s" % sp
+
+    # Get node names
+    get_node_names_cmd = (
+        "oc get node -o wide | grep -e '%s ' | awk '{print $1}'" % (
+            " ' -e '".join(gluster_pod_nodes_ips)))
+    gluster_pod_node_names = command.cmd_run(
+        get_node_names_cmd, hostname=ocp_node)
+    gluster_pod_node_names = [
+        node_name.strip()
+        for node_name in gluster_pod_node_names.split("\n")
+        if node_name.strip()
+    ]
+    node_count = len(gluster_pod_node_names)
+    err_msg = "Expected more than one node hosting Gluster PODs. Got '%s'." % (
+        node_count)
+    assert (node_count > 1), err_msg
+
+    # Get Gluster POD names which are located on the filtered nodes
+    get_pod_name_cmd = (
+        "oc get pods --all-namespaces "
+        "-o=custom-columns=:.metadata.name,:.spec.nodeName,:.status.hostIP | "
+        "grep 'glusterfs-' | grep -e '%s '" % "' -e '".join(
+            gluster_pod_node_names)
+    )
+    out = command.cmd_run(
+        get_pod_name_cmd, hostname=ocp_node)
+    data = []
+    for line in out.split("\n"):
+        pod_name, host_name, host_ip = [
+            el.strip() for el in line.split(" ") if el.strip()]
+        data.append({
+            "pod_name": pod_name,
+            "host_name": host_name,
+            "host_ip": host_ip,
+        })
+    pod_count = len(data)
+    err_msg = "Expected 3 or more Gluster PODs to be found. Actual is '%s'" % (
+        pod_count)
+    assert (pod_count > 2), err_msg
+    return data
+
+
+def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node=None):
+    """Run shell command on either Gluster PODs or Gluster nodes.
+
+    Args:
+        ocp_client_node (str): Node to execute OCP commands on.
+        cmd (str): shell command to run.
+        gluster_node (str): optional. Allows to chose specific gluster node,
+            keeping abstraction from deployment type. Can be either IP address
+            or node name from "oc get nodes" command.
+    Returns:
+        Output of a shell command as string object.
+    """
+    # Containerized Glusterfs
+    gluster_pods = oc_get_pods(ocp_client_node, selector="glusterfs-node=pod")
+    err_msg = ""
+    if gluster_pods:
+        if gluster_node:
+            for pod_name, pod_data in gluster_pods.items():
+                if gluster_node in (pod_data["ip"], pod_data["node"]):
+                    gluster_pod_names = [pod_name]
+                    break
+            else:
+                raise exceptions.ExecutionError(
+                    "Could not find Gluster PODs with node filter as "
+                    "'%s'." % gluster_node)
+        else:
+            gluster_pod_names = gluster_pods.keys()
+
+        for gluster_pod_name in gluster_pod_names:
+            try:
+                pod_cmd = "oc exec %s -- %s" % (gluster_pod_name, cmd)
+                return command.cmd_run(pod_cmd, hostname=ocp_client_node)
+            except Exception as e:
+                err = ("Failed to run '%s' command on '%s' Gluster POD. "
+                       "Error: %s\n" % (cmd, gluster_pod_name, e))
+                err_msg += err
+                g.log.error(err)
+        raise exceptions.ExecutionError(err_msg)
+
+    # Standalone Glusterfs
+    if gluster_node:
+        g_hosts = [gluster_node]
+    else:
+        g_hosts = g.config.get("gluster_servers", {}).keys()
+    for g_host in g_hosts:
+        try:
+            return command.cmd_run(cmd, hostname=g_host)
+        except Exception as e:
+            err = ("Failed to run '%s' command on '%s' Gluster node. "
+                   "Error: %s\n" % (cmd, g_host, e))
+            err_msg += err
+            g.log.error(err)
+
+    if not err_msg:
+        raise exceptions.ExecutionError(
+            "Haven't found neither Gluster PODs nor Gluster nodes.")
+    raise exceptions.ExecutionError(err_msg)
+
+
+def get_gluster_vol_info_by_pvc_name(ocp_node, pvc_name):
+    """Get Gluster volume info based on the PVC name.
+
+    Args:
+        ocp_node (str): Node to execute OCP commands on.
+        pvc_name (str): Name of a PVC to get bound Gluster volume info.
+    Returns:
+        dict: Dictionary containting data about a Gluster volume.
+    """
+
+    # Get PV ID from PVC
+    get_pvc_cmd = "oc get pvc %s -o=custom-columns=:.spec.volumeName" % (
+        pvc_name)
+    pv_name = command.cmd_run(get_pvc_cmd, hostname=ocp_node)
+    assert pv_name, "PV name should not be empty: '%s'" % pv_name
+
+    # Get volume ID from PV
+    get_pv_cmd = "oc get pv %s -o=custom-columns=:.spec.glusterfs.path" % (
+        pv_name)
+    vol_id = command.cmd_run(get_pv_cmd, hostname=ocp_node)
+    assert vol_id, "Gluster volume ID should not be empty: '%s'" % vol_id
+
+    vol_info_cmd = "gluster v info %s --xml" % vol_id
+    vol_info = cmd_run_on_gluster_pod_or_node(ocp_node, vol_info_cmd)
+
+    # Parse XML output to python dict
+    with mock.patch('glusto.core.Glusto.run', return_value=(0, vol_info, '')):
+        vol_info = volume_ops.get_volume_info(vol_id)
+        vol_info = vol_info[list(vol_info.keys())[0]]
+        vol_info["gluster_vol_id"] = vol_id
+        return vol_info
+
+
+def get_gluster_blockvol_info_by_pvc_name(ocp_node, heketi_server_url,
+                                          pvc_name):
+    """Get Gluster block volume info based on the PVC name.
+
+    Args:
+        ocp_node (str): Node to execute OCP commands on.
+        heketi_server_url (str): Heketi server url
+        pvc_name (str): Name of a PVC to get bound Gluster block volume info.
+    Returns:
+        dict: Dictionary containting data about a Gluster block volume.
+    """
+
+    # Get block volume Name and ID from PV which is bound to our PVC
+    get_block_vol_data_cmd = (
+        r'oc get pv --no-headers -o custom-columns='
+        r':.metadata.annotations.glusterBlockShare,'
+        r':.metadata.annotations."gluster\.org\/volume\-id",'
+        r':.spec.claimRef.name | grep "%s"' % pvc_name)
+    out = command.cmd_run(get_block_vol_data_cmd, hostname=ocp_node)
+    parsed_out = filter(None, map(str.strip, out.split(" ")))
+    assert len(parsed_out) == 3, "Expected 3 fields in following: %s" % out
+    block_vol_name, block_vol_id = parsed_out[:2]
+
+    # Get block hosting volume ID
+    block_hosting_vol_id = heketi_blockvolume_info(
+        ocp_node, heketi_server_url, block_vol_id, json=True
+    )["blockhostingvolume"]
+
+    # Get block hosting volume name by it's ID
+    block_hosting_vol_name = heketi_volume_info(
+        ocp_node, heketi_server_url, block_hosting_vol_id, json=True)['name']
+
+    # Get Gluster block volume info
+    vol_info_cmd = "gluster-block info %s/%s --json" % (
+        block_hosting_vol_name, block_vol_name)
+    vol_info = cmd_run_on_gluster_pod_or_node(ocp_node, vol_info_cmd)
+
+    return json.loads(vol_info)
+
+
+def wait_for_pod_be_ready(hostname, pod_name,
+                          timeout=1200, wait_step=60):
+    '''
+     This funciton waits for pod to be in ready state
+     Args:
+         hostname (str): hostname on which we want to check the pod status
+         pod_name (str): pod_name for which we need the status
+         timeout (int): timeout value,,
+                        default value is 1200 sec
+         wait_step( int): wait step,
+                          default value is 60 sec
+     Returns:
+         bool: True if pod status is Running and ready state,
+               otherwise Raise Exception
+    '''
+    for w in waiter.Waiter(timeout, wait_step):
+        # command to find pod status and its phase
+        cmd = ("oc get pods %s -o=custom-columns="
+               ":.status.containerStatuses[0].ready,"
+               ":.status.phase") % pod_name
+        ret, out, err = g.run(hostname, cmd, "root")
+        if ret != 0:
+            msg = "Failed to execute cmd: %s\nout: %s\nerr: %s" % (
+                cmd, out, err)
+            g.log.error(msg)
+            raise exceptions.ExecutionError(msg)
+        output = out.strip().split()
+
+        # command to find if pod is ready
+        if output[0] == "true" and output[1] == "Running":
+            g.log.info("pod %s is in ready state and is "
+                       "Running" % pod_name)
+            return True
+        elif output[1] == "Error":
+            msg = ("pod %s status error" % pod_name)
+            g.log.error(msg)
+            raise exceptions.ExecutionError(msg)
+        else:
+            g.log.info("pod %s ready state is %s,"
+                       " phase is %s,"
+                       " sleeping for %s sec" % (
+                           pod_name, output[0],
+                           output[1], wait_step))
+            continue
+    if w.expired:
+        err_msg = ("exceeded timeout %s for waiting for pod %s "
+                   "to be in ready state" % (timeout, pod_name))
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def get_pod_names_from_dc(hostname, dc_name, timeout=180, wait_step=3):
+    """Return list of POD names by their DC.
+
+    Args:
+        hostname (str): hostname on which 'oc' commands will be executed.
+        dc_name (str): deployment_confidg name
+        timeout (int): timeout value. Default value is 180 sec.
+        wait_step( int): Wait step, default value is 3 sec.
+    Returns:
+         list: list of strings which are POD names
+    Raises: exceptions.ExecutionError
+    """
+    get_replicas_amount_cmd = (
+        "oc get dc --no-headers --all-namespaces "
+        "-o=custom-columns=:.spec.replicas,:.metadata.name "
+        "| grep '%s' | awk '{print $1}'" % dc_name)
+    replicas = int(command.cmd_run(
+        get_replicas_amount_cmd, hostname=hostname))
+
+    get_pod_names_cmd = (
+        "oc get pods --all-namespaces -o=custom-columns=:.metadata.name "
+        "--no-headers=true --selector deploymentconfig=%s" % dc_name)
+    for w in waiter.Waiter(timeout, wait_step):
+        out = command.cmd_run(get_pod_names_cmd, hostname=hostname)
+        pod_names = [o.strip() for o in out.split('\n') if o.strip()]
+        if len(pod_names) != replicas:
+            continue
+        g.log.info(
+            "POD names for '%s' DC are '%s'. "
+            "Expected amount of PODs is '%s'.", dc_name, out, replicas)
+        return pod_names
+    if w.expired:
+        err_msg = ("Exceeded %s sec timeout waiting for PODs to appear "
+                   "in amount of %s." % (timeout, replicas))
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def get_pod_name_from_dc(hostname, dc_name, timeout=180, wait_step=3):
+    return get_pod_names_from_dc(
+        hostname, dc_name, timeout=timeout, wait_step=wait_step)[0]
+
+
+def get_pvc_status(hostname, pvc_name):
+    '''
+     This function verifies the if pod is running
+     Args:
+         hostname (str): hostname on which we want
+                         to check the pvc status
+         pvc_name (str): pod_name for which we
+                         need the status
+     Returns:
+         bool, status (str): True, status of pvc
+               otherwise False, error message.
+    '''
+    cmd = "oc get pvc | grep %s | awk '{print $2}'" % pvc_name
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret != 0:
+        g.log.error("failed to execute cmd %s" % cmd)
+        return False, err
+    output = out.strip().split("\n")[0].strip()
+    return True, output
+
+
+def verify_pvc_status_is_bound(hostname, pvc_name, timeout=120, wait_step=3):
+    """Verify that PVC gets 'Bound' status in required time.
+
+    Args:
+        hostname (str): hostname on which we will execute oc commands
+        pvc_name (str): name of PVC to check status of
+        timeout (int): total time in seconds we are ok to wait
+                       for 'Bound' status of a PVC
+        wait_step (int): time in seconds we will sleep before checking a PVC
+                         status again.
+    Returns: None
+    Raises: exceptions.ExecutionError in case of errors.
+    """
+    pvc_not_found_counter = 0
+    for w in waiter.Waiter(timeout, wait_step):
+        ret, output = get_pvc_status(hostname, pvc_name)
+        if ret is not True:
+            msg = ("Failed to execute 'get' command for '%s' PVC. "
+                   "Got following responce: %s" % (pvc_name, output))
+            g.log.error(msg)
+            raise exceptions.ExecutionError(msg)
+        if output == "":
+            g.log.info("PVC '%s' not found, sleeping for %s "
+                       "sec." % (pvc_name, wait_step))
+            if pvc_not_found_counter > 0:
+                msg = ("PVC '%s' has not been found 2 times already. "
+                       "Make sure you provided correct PVC name." % pvc_name)
+            else:
+                pvc_not_found_counter += 1
+                continue
+        elif output == "Pending":
+            g.log.info("PVC '%s' is in Pending state, sleeping for %s "
+                       "sec" % (pvc_name, wait_step))
+            continue
+        elif output == "Bound":
+            g.log.info("PVC '%s' is in Bound state." % pvc_name)
+            return pvc_name
+        elif output == "Error":
+            msg = "PVC '%s' is in 'Error' state." % pvc_name
+            g.log.error(msg)
+        else:
+            msg = "PVC %s has different status - %s" % (pvc_name, output)
+            g.log.error(msg)
+        if msg:
+            raise AssertionError(msg)
+    if w.expired:
+        msg = ("Exceeded timeout of '%s' seconds for verifying PVC '%s' "
+               "to reach the 'Bound' state." % (timeout, pvc_name))
+
+        # Gather more info for ease of debugging
+        try:
+            pvc_events = get_events(hostname, obj_name=pvc_name)
+        except Exception:
+            pvc_events = '?'
+        msg += "\nPVC events: %s" % pvc_events
+
+        g.log.error(msg)
+        raise AssertionError(msg)
+
+
+def resize_pvc(hostname, pvc_name, size):
+    '''
+     Resize PVC
+     Args:
+         hostname (str): hostname on which we want
+                         to edit the pvc status
+         pvc_name (str): pod_name for which we
+                         edit the storage capacity
+         size (int): size of pvc to change
+     Returns:
+         bool: True, if successful
+               otherwise raise Exception
+    '''
+    cmd = ("oc patch  pvc %s "
+           "-p='{\"spec\": {\"resources\": {\"requests\": "
+           "{\"storage\": \"%dGi\"}}}}'" % (pvc_name, size))
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret != 0:
+        error_msg = ("failed to execute cmd %s "
+                     "out- %s err %s" % (cmd, out, err))
+        g.log.error(error_msg)
+        raise exceptions.ExecutionError(error_msg)
+
+    g.log.info("successfully edited storage capacity"
+               "of pvc %s . out- %s" % (pvc_name, out))
+    return True
+
+
+def verify_pvc_size(hostname, pvc_name, size,
+                    timeout=120, wait_step=5):
+    '''
+     Verify size of PVC
+     Args:
+         hostname (str): hostname on which we want
+                         to verify the size of pvc
+         pvc_name (str): pvc_name for which we
+                         verify its size
+         size (int): size of pvc
+         timeout (int): timeout value,
+                        verifies the size after wait_step
+                        value till timeout
+                        default value is 120 sec
+         wait_step( int): wait step,
+                          default value is 5 sec
+     Returns:
+         bool: True, if successful
+               otherwise raise Exception
+    '''
+    cmd = ("oc get pvc %s -o=custom-columns="
+           ":.spec.resources.requests.storage,"
+           ":.status.capacity.storage" % pvc_name)
+    for w in waiter.Waiter(timeout, wait_step):
+        sizes = command.cmd_run(cmd, hostname=hostname).split()
+        spec_size = int(sizes[0].replace("Gi", ""))
+        actual_size = int(sizes[1].replace("Gi", ""))
+        if spec_size == actual_size == size:
+            g.log.info("verification of pvc %s of size %d "
+                       "successful" % (pvc_name, size))
+            return True
+        else:
+            g.log.info("sleeping for %s sec" % wait_step)
+            continue
+
+    err_msg = ("verification of pvc %s size of %d failed -"
+               "spec_size- %d actual_size %d" % (
+                   pvc_name, size, spec_size, actual_size))
+    g.log.error(err_msg)
+    raise AssertionError(err_msg)
+
+
+def verify_pv_size(hostname, pv_name, size,
+                   timeout=120, wait_step=5):
+    '''
+     Verify size of PV
+     Args:
+         hostname (str): hostname on which we want
+                         to verify the size of pv
+         pv_name (str): pv_name for which we
+                         verify its size
+         size (int): size of pv
+         timeout (int): timeout value,
+                        verifies the size after wait_step
+                        value till timeout
+                        default value is 120 sec
+         wait_step( int): wait step,
+                          default value is 5 sec
+     Returns:
+         bool: True, if successful
+               otherwise raise Exception
+    '''
+    cmd = ("oc get pv %s -o=custom-columns=:."
+           "spec.capacity.storage" % pv_name)
+    for w in waiter.Waiter(timeout, wait_step):
+        pv_size = command.cmd_run(cmd, hostname=hostname).split()[0]
+        pv_size = int(pv_size.replace("Gi", ""))
+        if pv_size == size:
+            g.log.info("verification of pv %s of size %d "
+                       "successful" % (pv_name, size))
+            return True
+        else:
+            g.log.info("sleeping for %s sec" % wait_step)
+            continue
+
+    err_msg = ("verification of pv %s size of %d failed -"
+               "pv_size- %d" % (pv_name, size, pv_size))
+    g.log.error(err_msg)
+    raise AssertionError(err_msg)
+
+
+def get_pv_name_from_pvc(hostname, pvc_name):
+    '''
+     Returns PV name of the corresponding PVC name
+     Args:
+         hostname (str): hostname on which we want
+                         to find pv name
+         pvc_name (str): pvc_name for which we
+                         want to find corresponding
+                         pv name
+     Returns:
+         pv_name (str): pv name if successful,
+                        otherwise raise Exception
+    '''
+    cmd = ("oc get pvc %s -o=custom-columns=:."
+           "spec.volumeName" % pvc_name)
+    pv_name = command.cmd_run(cmd, hostname=hostname)
+    g.log.info("pv name is %s for pvc %s" % (
+                   pv_name, pvc_name))
+
+    return pv_name
+
+
+def get_vol_names_from_pv(hostname, pv_name):
+    '''
+     Returns the heketi and gluster
+     vol names of the corresponding PV
+     Args:
+         hostname (str): hostname on which we want
+                         to find vol names
+         pv_name (str): pv_name for which we
+                        want to find corresponding
+                        vol names
+     Returns:
+         volname (dict): dict if successful
+                      {"heketi_vol": heketi_vol_name,
+                       "gluster_vol": gluster_vol_name
+                    ex: {"heketi_vol": " xxxx",
+                         "gluster_vol": "vol_xxxx"]
+                    otherwise raise Exception
+    '''
+    vol_dict = {}
+    cmd = (r"oc get pv %s -o=custom-columns="
+           r":.metadata.annotations."
+           r"'gluster\.kubernetes\.io\/heketi\-volume\-id',"
+           r":.spec.glusterfs.path" % pv_name)
+    vol_list = command.cmd_run(cmd, hostname=hostname).split()
+    vol_dict = {"heketi_vol": vol_list[0],
+                "gluster_vol": vol_list[1]}
+    g.log.info("gluster vol name is %s and heketi vol name"
+               " is %s for pv %s"
+               % (vol_list[1], vol_list[0], pv_name))
+    return vol_dict
+
+
+def get_events(hostname,
+               obj_name=None, obj_namespace=None, obj_type=None,
+               event_reason=None, event_type=None):
+    """Return filtered list of events.
+
+    Args:
+        hostname (str): hostname of oc client
+        obj_name (str): name of an object
+        obj_namespace (str): namespace where object is located
+        obj_type (str): type of an object, i.e. PersistentVolumeClaim or Pod
+        event_reason (str): reason why event was created,
+            i.e. Created, Started, Unhealthy, SuccessfulCreate, Scheduled ...
+        event_type (str): type of an event, i.e. Normal or Warning
+    Returns:
+        List of dictionaries, where the latter are of following structure:
+        {
+            "involvedObject": {
+                "kind": "ReplicationController",
+                "name": "foo-object-name",
+                "namespace": "foo-object-namespace",
+            },
+            "message": "Created pod: foo-object-name",
+            "metadata": {
+                "creationTimestamp": "2018-10-19T18:27:09Z",
+                "name": "foo-object-name.155f15db4e72cc2e",
+                "namespace": "foo-object-namespace",
+            },
+            "reason": "SuccessfulCreate",
+            "reportingComponent": "",
+            "reportingInstance": "",
+            "source": {"component": "replication-controller"},
+            "type": "Normal"
+        }
+    """
+    field_selector = []
+    if obj_name:
+        field_selector.append('involvedObject.name=%s' % obj_name)
+    if obj_namespace:
+        field_selector.append('involvedObject.namespace=%s' % obj_namespace)
+    if obj_type:
+        field_selector.append('involvedObject.kind=%s' % obj_type)
+    if event_reason:
+        field_selector.append('reason=%s' % event_reason)
+    if event_type:
+        field_selector.append('type=%s' % event_type)
+    cmd = "oc get events -o yaml --field-selector %s" % ",".join(
+        field_selector or "''")
+    return yaml.load(command.cmd_run(cmd, hostname=hostname))['items']
+
+
+def wait_for_events(hostname,
+                    obj_name=None, obj_namespace=None, obj_type=None,
+                    event_reason=None, event_type=None,
+                    timeout=120, wait_step=3):
+    """Wait for appearence of specific set of events."""
+    for w in waiter.Waiter(timeout, wait_step):
+        events = get_events(
+            hostname=hostname, obj_name=obj_name, obj_namespace=obj_namespace,
+            obj_type=obj_type, event_reason=event_reason,
+            event_type=event_type)
+        if events:
+            return events
+    if w.expired:
+        err_msg = ("Exceeded %ssec timeout waiting for events." % timeout)
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def match_pvc_and_pv(hostname, prefix):
+    """Match OCP PVCs and PVs generated
+
+    Args:
+        hostname (str): hostname of oc client
+        prefix (str): pv prefix used by user at time
+                      of pvc creation
+    """
+    pvc_list = sorted([
+        pvc[0]
+        for pvc in oc_get_custom_resource(hostname, "pvc", ":.metadata.name")
+        if pvc[0].startswith(prefix)
+    ])
+
+    pv_list = sorted([
+        pv[0]
+        for pv in oc_get_custom_resource(
+            hostname, "pv", ":.spec.claimRef.name"
+        )
+        if pv[0].startswith(prefix)
+    ])
+
+    if cmp(pvc_list, pv_list) != 0:
+        err_msg = "PVC and PV list match failed"
+        err_msg += "\nPVC list: %s, " % pvc_list
+        err_msg += "\nPV list %s" % pv_list
+        err_msg += "\nDifference: %s" % (set(pvc_list) ^ set(pv_list))
+        raise AssertionError(err_msg)
+
+
+def match_pv_and_heketi_block_volumes(
+        hostname, heketi_block_volumes, pvc_prefix):
+    """Match heketi block volumes and OC PVCs
+
+    Args:
+        hostname (str): hostname on which we want to check heketi
+                        block volumes and OCP PVCs
+        heketi_block_volumes (list): list of heketi block volume names
+        pvc_prefix (str): pv prefix given by user at the time of pvc creation
+    """
+    custom_columns = [
+        r':.spec.claimRef.name',
+        r':.metadata.annotations."pv\.kubernetes\.io\/provisioned\-by"',
+        r':.metadata.annotations."gluster\.org\/volume\-id"'
+    ]
+    pv_block_volumes = sorted([
+        pv[2]
+        for pv in oc_get_custom_resource(hostname, "pv", custom_columns)
+        if pv[0].startswith(pvc_prefix) and pv[1] == "gluster.org/glusterblock"
+    ])
+
+    if cmp(pv_block_volumes, heketi_block_volumes) != 0:
+        err_msg = "PV block volumes and Heketi Block volume list match failed"
+        err_msg += "\nPV Block Volumes: %s, " % pv_block_volumes
+        err_msg += "\nHeketi Block volumes %s" % heketi_block_volumes
+        err_msg += "\nDifference: %s" % (set(pv_block_volumes) ^
+                                         set(heketi_block_volumes))
+        raise AssertionError(err_msg)
+
+
+def check_service_status_on_pod(
+        ocp_client, podname, service, status, timeout=180, wait_step=3):
+    """Check a service state on a pod.
+
+    Args:
+        ocp_client (str): node with 'oc' client
+        podname (str): pod name on which service needs to be checked
+        service (str): service which needs to be checked
+        status (str): status to be checked
+        timeout (int): seconds to wait before service starts having
+                       specified 'status'
+        wait_step (int): interval in seconds to wait before checking
+                         service again.
+    """
+    err_msg = ("Exceeded timeout of %s sec for verifying %s service to start "
+               "having '%s' status" % (timeout, service, status))
+
+    for w in waiter.Waiter(timeout, wait_step):
+        ret, out, err = oc_rsh(ocp_client, podname, SERVICE_STATUS % service)
+        if ret != 0:
+            err_msg = ("failed to get service %s's status on pod %s" %
+                       (service, podname))
+            g.log.error(err_msg)
+            raise AssertionError(err_msg)
+
+        for line in out.splitlines():
+            status_match = re.search(SERVICE_STATUS_REGEX, line)
+            if status_match and status_match.group(1) == status:
+                return True
+
+    if w.expired:
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def wait_for_service_status_on_gluster_pod_or_node(
+        ocp_client, service, status, gluster_node, timeout=180, wait_step=3):
+    """Wait for a service specific status on a Gluster POD or node.
+
+    Args:
+        ocp_client (str): hostname on which we want to check service
+        service (str): target service to be checked
+        status (str): service status which we wait for
+        gluster_node (str): Gluster node IPv4 which stores either Gluster POD
+            or Gluster services directly.
+        timeout (int): seconds to wait before service starts having
+                       specified 'status'
+        wait_step (int): interval in seconds to wait before checking
+                         service again.
+    """
+    err_msg = ("Exceeded timeout of %s sec for verifying %s service to start "
+               "having '%s' status" % (timeout, service, status))
+
+    for w in waiter.Waiter(timeout, wait_step):
+        out = cmd_run_on_gluster_pod_or_node(
+            ocp_client, SERVICE_STATUS % service, gluster_node)
+        for line in out.splitlines():
+            status_match = re.search(SERVICE_STATUS_REGEX, line)
+            if status_match and status_match.group(1) == status:
+                return True
+    if w.expired:
+        g.log.error(err_msg)
+        raise exceptions.ExecutionError(err_msg)
+
+
+def restart_service_on_gluster_pod_or_node(ocp_client, service, gluster_node):
+    """Restart service on Gluster either POD or node.
+
+    Args:
+        ocp_client (str): host on which we want to run 'oc' commands.
+        service (str): service which needs to be restarted
+        gluster_node (str): Gluster node IPv4 which stores either Gluster POD
+            or Gluster services directly.
+    Raises:
+        AssertionError in case restart of a service fails.
+    """
+    cmd_run_on_gluster_pod_or_node(
+        ocp_client, SERVICE_RESTART % service, gluster_node)
+
+
+def oc_adm_manage_node(
+        ocp_client, operation, nodes=None, node_selector=None):
+    """Manage common operations on nodes for administrators.
+
+    Args:
+        ocp_client (str): host on which we want to run 'oc' commands.
+        operations (str):
+            eg. --schedulable=true.
+        nodes (list): list of nodes to manage.
+        node_selector (str): selector to select the nodes.
+            Note: 'nodes' and 'node_selector' are are mutually exclusive.
+            Only either of them should be passed as parameter not both.
+    Returns:
+        str: In case of success.
+    Raises:
+        AssertionError: In case of any failures.
+    """
+
+    if (not nodes) == (not node_selector):
+        raise AssertionError(
+            "'nodes' and 'node_selector' are mutually exclusive. "
+            "Only either of them should be passed as parameter not both.")
+
+    cmd = "oc adm manage-node %s" % operation
+    if node_selector:
+        cmd += " --selector %s" % node_selector
+    else:
+        node = ' '.join(nodes)
+        cmd += " " + node
+
+    return command.cmd_run(cmd, ocp_client)
+
+
+def oc_get_schedulable_nodes(ocp_client):
+    """Get the list of schedulable nodes.
+
+    Args:
+        ocp_client (str): host on which we want to run 'oc' commands.
+
+    Returns:
+        list: list of nodes if present.
+    Raises:
+        AssertionError: In case of any failures.
+    """
+    cmd = ("oc get nodes --field-selector=spec.unschedulable!=true "
+           "-o=custom-columns=:.metadata.name,:.spec.taints[*].effect "
+           "--no-headers | awk '!/NoSchedule/{print $1}'")
+
+    out = command.cmd_run(cmd, ocp_client)
+
+    return out.split('\n') if out else out
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_storage_libs.py b/openshift-storage-libs/openshiftstoragelibs/openshift_storage_libs.py
new file mode 100644
index 00000000..4d2b4f61
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_storage_libs.py
@@ -0,0 +1,228 @@
+from glusto.core import Glusto as g
+import yaml
+
+from openshiftstoragelibs.command import cmd_run
+from openshiftstoragelibs.exceptions import (
+    ExecutionError,
+    NotSupportedException,
+)
+from openshiftstoragelibs.openshift_version import get_openshift_version
+
+
+MASTER_CONFIG_FILEPATH = "/etc/origin/master/master-config.yaml"
+
+
+def validate_multipath_pod(hostname, podname, hacount, mpath=""):
+    '''
+     This function validates multipath for given app-pod
+     Args:
+         hostname (str): ocp master node name
+         podname (str): app-pod name for which we need to validate
+                        multipath. ex : nginx1
+         hacount (int): multipath count or HA count. ex: 3
+     Returns:
+         bool: True if successful,
+               otherwise False
+    '''
+    cmd = "oc get pods -o wide | grep %s | awk '{print $7}'" % podname
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret != 0 or out == "":
+        g.log.error("failed to exectute cmd %s on %s, err %s"
+                    % (cmd, hostname, out))
+        return False
+    pod_nodename = out.strip()
+    active_node_count = 1
+    enable_node_count = hacount - 1
+    cmd = "multipath -ll %s | grep 'status=active' | wc -l" % mpath
+    ret, out, err = g.run(pod_nodename, cmd, "root")
+    if ret != 0 or out == "":
+        g.log.error("failed to exectute cmd %s on %s, err %s"
+                    % (cmd, pod_nodename, out))
+        return False
+    active_count = int(out.strip())
+    if active_node_count != active_count:
+        g.log.error("active node count on %s for %s is %s and not 1"
+                    % (pod_nodename, podname, active_count))
+        return False
+    cmd = "multipath -ll %s | grep 'status=enabled' | wc -l" % mpath
+    ret, out, err = g.run(pod_nodename, cmd, "root")
+    if ret != 0 or out == "":
+        g.log.error("failed to exectute cmd %s on %s, err %s"
+                    % (cmd, pod_nodename, out))
+        return False
+    enable_count = int(out.strip())
+    if enable_node_count != enable_count:
+        g.log.error("passive node count on %s for %s is %s "
+                    "and not %s" % (
+                        pod_nodename, podname, enable_count,
+                        enable_node_count))
+        return False
+
+    g.log.info("validation of multipath for %s is successfull"
+               % podname)
+    return True
+
+
+def enable_pvc_resize(master_node):
+    '''
+     This function edits the /etc/origin/master/master-config.yaml
+     file - to enable pv_resize feature
+     and restarts atomic-openshift service on master node
+     Args:
+         master_node (str): hostname of masternode  on which
+                           want to edit the
+                           master-config.yaml file
+     Returns:
+         bool: True if successful,
+               otherwise raise Exception
+    '''
+    version = get_openshift_version()
+    if version < "3.9":
+        msg = ("pv resize is not available in openshift "
+               "version %s " % version)
+        g.log.error(msg)
+        raise NotSupportedException(msg)
+
+    try:
+        conn = g.rpyc_get_connection(master_node, user="root")
+        if conn is None:
+            err_msg = ("Failed to get rpyc connection of node %s"
+                       % master_node)
+            g.log.error(err_msg)
+            raise ExecutionError(err_msg)
+
+        with conn.builtin.open(MASTER_CONFIG_FILEPATH, 'r') as f:
+            data = yaml.load(f)
+            dict_add = data['admissionConfig']['pluginConfig']
+            if "PersistentVolumeClaimResize" in dict_add:
+                g.log.info("master-config.yaml file is already edited")
+                return True
+            dict_add['PersistentVolumeClaimResize'] = {
+                'configuration': {
+                    'apiVersion': 'v1',
+                    'disable': 'false',
+                    'kind': 'DefaultAdmissionConfig'}}
+            data['admissionConfig']['pluginConfig'] = dict_add
+            kube_config = data['kubernetesMasterConfig']
+            for key in ('apiServerArguments', 'controllerArguments'):
+                kube_config[key] = (
+                    kube_config.get(key)
+                    if isinstance(kube_config.get(key), dict) else {})
+                value = ['ExpandPersistentVolumes=true']
+                kube_config[key]['feature-gates'] = value
+        with conn.builtin.open(MASTER_CONFIG_FILEPATH, 'w+') as f:
+            yaml.dump(data, f, default_flow_style=False)
+    except Exception as err:
+        raise ExecutionError("failed to edit master-config.yaml file "
+                             "%s on %s" % (err, master_node))
+    finally:
+        g.rpyc_close_connection(master_node, user="root")
+
+    g.log.info("successfully edited master-config.yaml file "
+               "%s" % master_node)
+    if version == "3.9":
+        cmd = ("systemctl restart atomic-openshift-master-api "
+               "atomic-openshift-master-controllers")
+    else:
+        cmd = ("/usr/local/bin/master-restart api && "
+               "/usr/local/bin/master-restart controllers")
+    ret, out, err = g.run(master_node, cmd, "root")
+    if ret != 0:
+        err_msg = "Failed to execute cmd %s on %s\nout: %s\nerr: %s" % (
+            cmd, master_node, out, err)
+        g.log.error(err_msg)
+        raise ExecutionError(err_msg)
+
+    return True
+
+
+def get_iscsi_session(node, iqn=None, raise_on_error=True):
+    """Get the list of ip's of iscsi sessions.
+
+    Args:
+        node (str): where we want to run the command.
+        iqn (str): name of iqn.
+    Returns:
+        list: list of session ip's.
+    raises:
+        ExecutionError: In case of any failure if raise_on_error=True.
+    """
+
+    cmd = "set -o pipefail && ((iscsiadm -m session"
+    if iqn:
+        cmd += " | grep %s" % iqn
+    cmd += ") | awk '{print $3}' | cut -d ':' -f 1)"
+
+    out = cmd_run(cmd, node, raise_on_error=raise_on_error)
+
+    return out.split("\n") if out else out
+
+
+def get_iscsi_block_devices_by_path(node, iqn=None, raise_on_error=True):
+    """Get list of iscsiadm block devices from path.
+
+    Args:
+        node (str): where we want to run the command.
+        iqn (str): name of iqn.
+    returns:
+        dictionary: block devices and there ips.
+    raises:
+        ExecutionError: In case of any failure if raise_on_error=True.
+    """
+    cmd = "set -o pipefail && ((ls --format=context /dev/disk/by-path/ip*"
+    if iqn:
+        cmd += " | grep %s" % iqn
+    cmd += ") | awk -F '/|:|-' '{print $10,$25}')"
+
+    out = cmd_run(cmd, node, raise_on_error=raise_on_error)
+
+    if not out:
+        return out
+
+    out_dic = {}
+    for i in out.split("\n"):
+        ip, device = i.strip().split(" ")
+        out_dic[device] = ip
+
+    return out_dic
+
+
+def get_mpath_name_from_device_name(node, device, raise_on_error=True):
+    """Get name of mpath device form block device
+
+    Args:
+        node (str): where we want to run the command.
+        device (str): for which we have to find mpath.
+    Returns:
+        str: name of device
+    Raises:
+        ExecutionError: In case of any failure if raise_on_error=True.
+    """
+    cmd = ("set -o pipefail && ((lsblk -n --list --output=NAME /dev/%s)"
+           " | tail -1)" % device)
+
+    return cmd_run(cmd, node, raise_on_error=raise_on_error)
+
+
+def get_active_and_enabled_devices_from_mpath(node, mpath):
+    """Get active and enabled devices from mpath name.
+
+    Args:
+        node (str): where we want to run the command.
+        mpath (str): name of mpath for which we have to find devices.
+    Returns:
+        dictionary: devices info
+    Raises:
+        ExecutionError: In case of any failure
+    """
+
+    cmd = ("set -o pipefail && ((multipath -ll %s | grep -A 1 status=%s)"
+           " | cut -d ':' -f 4 | awk '{print $2}')")
+
+    active = cmd_run(cmd % (mpath, 'active'), node).split('\n')[1::2]
+    enabled = cmd_run(cmd % (mpath, 'enabled'), node).split('\n')[1::2]
+
+    out_dic = {
+        'active': active,
+        'enabled': enabled}
+    return out_dic
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_version.py b/openshift-storage-libs/openshiftstoragelibs/openshift_version.py
new file mode 100644
index 00000000..bc0c9be0
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_version.py
@@ -0,0 +1,173 @@
+"""
+Use this module for any OpenShift version comparisons.
+
+Usage example:
+
+    # Assume OpenShift version is '3.10.45'. Then we have following:
+    from openshiftstoragelibs import openshift_version
+    version = openshift_version.get_openshift_version()
+    if version < '3.10':
+        # False
+    if version <= '3.10':
+        # True
+    if version < '3.10.46':
+        # True
+    if version < '3.10.13':
+        # False
+    if '3.9' < version <= '3.11':
+        # True
+
+Notes:
+- If one of comparison operands has empty/zero-like 'micro' part of version,
+  then it is ignored during comparison, where only 'major' and 'minor' parts of
+  the OpenShift versions are used.
+
+"""
+import re
+
+from glusto.core import Glusto as g
+import six
+
+from openshiftstoragelibs import exceptions
+
+
+OPENSHIFT_VERSION_RE = r"(?:v?)(\d+)(?:\.)(\d+)(?:\.(\d+))?$"
+OPENSHIFT_VERSION = None
+
+
+def _get_openshift_version_str(hostname=None):
+    """Gets OpenShift version from 'oc version' command.
+
+    Args:
+        hostname (str): Node on which the ocp command should run.
+    Returns:
+        str : oc version, i.e. 'v3.10.47'
+    Raises: 'exceptions.ExecutionError' if failed to get version
+    """
+    if not hostname:
+        hostname = list(g.config['ocp_servers']['client'].keys())[0]
+    cmd = "oc version | grep openshift | cut -d ' ' -f 2"
+    ret, out, err = g.run(hostname, cmd, "root")
+    if ret != 0:
+        msg = "Failed to get oc version. \n'err': %s\n 'out': %s" % (err, out)
+        g.log.error(msg)
+        raise AssertionError(msg)
+    out = out.strip()
+    if not out:
+        error_msg = "Empty output from 'oc version' command: '%s'" % out
+        g.log.error(error_msg)
+        raise exceptions.ExecutionError(error_msg)
+
+    return out
+
+
+def _parse_openshift_version(openshift_version_str):
+    """Parses OpenShift version str into tuple of 3 values.
+
+    Args:
+        openshift_version_str (str): OpenShift version like '3.10' or '3.10.45'
+    Returns:
+        Tuple object of 3 values - major, minor and micro version parts.
+    """
+    groups = re.findall(OPENSHIFT_VERSION_RE, openshift_version_str)
+    err_msg = (
+        "Failed to parse '%s' str into 3 OpenShift version parts - "
+        "'major', 'minor' and 'micro'. "
+        "Expected value like '3.10' or '3.10.45'" % openshift_version_str)
+    assert groups, err_msg
+    assert len(groups) == 1, err_msg
+    assert len(groups[0]) == 3, err_msg
+    return (int(groups[0][0]), int(groups[0][1]), int(groups[0][2] or 0))
+
+
+class OpenshiftVersion(object):
+    """Eases OpenShift versions comparison.
+
+    Instance of this class can be used for comparison with other instance of
+    it or to string-like objects.
+
+    Input str version is required to have, at least, 2 version parts -
+    'major' and 'minor'. Third part is optional - 'micro' version.
+    Examples: '3.10', 'v3.10', '3.10.45', 'v3.10.45'.
+
+    Before each comparison, both operands are checked for zero value in 'micro'
+    part. If one or both are false, then 'micro' part not used for comparison.
+
+    Usage example (1) - compare to string object:
+        version_3_10 = OpenshiftVersion('3.10')
+        cmp_result = '3.9' < version_3_10 <= '3.11'
+
+    Usage example (2) - compare to the same type of an object:
+        version_3_10 = OpenshiftVersion('3.10')
+        version_3_11 = OpenshiftVersion('3.11')
+        cmp_result = version_3_10 < version_3_11
+    """
+    def __init__(self, openshift_version_str):
+        self.v = _parse_openshift_version(openshift_version_str)
+        self.major, self.minor, self.micro = self.v
+
+    def _adapt_other(self, other):
+        if isinstance(other, six.string_types):
+            return OpenshiftVersion(other)
+        elif isinstance(other, OpenshiftVersion):
+            return other
+        else:
+            raise NotImplementedError(
+                "'%s' type is not supported for OpenShift version "
+                "comparison." % type(other))
+
+    def __lt__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] < adapted_other.v[0:2]
+        return self.v < adapted_other.v
+
+    def __le__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] <= adapted_other.v[0:2]
+        return self.v <= adapted_other.v
+
+    def __eq__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] == adapted_other.v[0:2]
+        return self.v == adapted_other.v
+
+    def __ge__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] >= adapted_other.v[0:2]
+        return self.v >= adapted_other.v
+
+    def __gt__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] > adapted_other.v[0:2]
+        return self.v > adapted_other.v
+
+    def __ne__(self, other):
+        adapted_other = self._adapt_other(other)
+        if not all((self.micro, adapted_other.micro)):
+            return self.v[0:2] != adapted_other.v[0:2]
+        return self.v != adapted_other.v
+
+
+def get_openshift_version(hostname=None):
+    """Cacher of an OpenShift version.
+
+    Version of an OpenShift cluster is constant value. So, we call API just
+    once and then reuse it's output.
+
+    Args:
+        hostname (str): a node with 'oc' client where command should run on.
+            If not specified, then first key
+            from 'ocp_servers.client' config option will be picked up.
+    Returns:
+        OpenshiftVersion object instance.
+    """
+    global OPENSHIFT_VERSION
+    if not OPENSHIFT_VERSION:
+        version_str = _get_openshift_version_str(hostname=hostname)
+        OPENSHIFT_VERSION = OpenshiftVersion(version_str)
+    return OPENSHIFT_VERSION
diff --git a/openshift-storage-libs/openshiftstoragelibs/podcmd.py b/openshift-storage-libs/openshiftstoragelibs/podcmd.py
new file mode 100644
index 00000000..bf84a8b9
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/podcmd.py
@@ -0,0 +1,141 @@
+"""Convenience wrappers for running commands within a pod
+
+The goal of this module is to support running glusto commands in pods
+without a lot of boilerplate and hassle. The basic idea is that we
+have our own run() function that can be addressed to a pod using the
+Pod object (namedtuple). This run function will work like a normal
+g.run() when not using the Pod object.
+
+Example:
+    >>> run("my-host.local", ["parted", "/dev/sda", "p"])
+    0, "<...stdout...>", "<...stderr...>"
+
+    >>> run(Pod("my-host.local", "my-pod-426ln"),
+    ...     ["pvs"])
+    0, "<...stdout...>", "<...stderr...>"
+
+In addition, if there's a need to to use some higher level functions
+that directly call into glusto run we can monkey-patch the glusto object
+using the GlustoPod context manager. GlustoPod can also be used as a
+decorator.
+
+Imagine a function that direcly calls g.run:
+    >>> def get_magic_number(host, ticket):
+    ...     s, out, _ = g.run(host, ['magicall', '--ticket', ticket])
+    ...     if s != 0:
+    ...         return None
+    ...     return out.strip()
+
+If you want to have this operate within a pod you can use the GlustoPod
+manager to enable the pod-aware run method and pass it a Pod object
+as the first argument. Example:
+    >>> def check_magic_number(ticket):
+    ...     with GlustoPod():
+    ...         m = get_magic_number(Pod('myhost', 'mypod'), ticket)
+    ...     return m > 42
+
+Similarly it can be used as a context manager:
+    >>> @GlustoPod()
+    ... def funky(x):
+    ...     m = get_magic_number(Pod('myhost', 'mypod'), ticket)
+    ...     return m > 42
+
+Because the custom run fuction only runs commands in pods when passed
+a Pod object it is fairly safe to enable the monkey-patch over the
+lifetime of a function that addresses both hosts and pods.
+"""
+
+from collections import namedtuple
+from functools import partial, wraps
+import types
+
+from glusto.core import Glusto as g
+
+from openshiftstoragelibs import openshift_ops
+
+# Define a namedtuple that allows us to address pods instead of just
+# hosts,
+Pod = namedtuple('Pod', 'node podname')
+
+
+def run(target, command, log_level=None, orig_run=g.run):
+    """Function that runs a command on a host or in a pod via a host.
+    Wraps glusto's run function.
+
+    Args:
+        target (str|Pod): If target is str object and
+            it equals to 'auto_get_gluster_endpoint', then
+            Gluster endpoint gets autocalculated to be any of
+            Gluster PODs or nodes depending on the deployment type of
+            a Gluster cluster.
+            If it is str object with other value, then it is considered to be
+            an endpoint for command.
+            If 'target' is of the 'Pod' type,
+            then command will run on the specified POD.
+        command (str|list): Command to run.
+        log_level (str|None): log level to be passed on to glusto's
+            run method
+        orig_run (function): The default implementation of the
+            run method. Will be used when target is not a pod.
+
+    Returns:
+        A tuple of the command's return code, stdout, and stderr.
+    """
+    # NOTE: orig_run captures the glusto run method at function
+    # definition time in order to capture the method before
+    # any additional monkeypatching by other code
+
+    if target == 'auto_get_gluster_endpoint':
+        ocp_client_node = list(g.config['ocp_servers']['client'].keys())[0]
+        gluster_pods = openshift_ops.get_ocp_gluster_pod_names(ocp_client_node)
+        if gluster_pods:
+            target = Pod(ocp_client_node, gluster_pods[0])
+        else:
+            target = list(g.config.get("gluster_servers", {}).keys())[0]
+
+    if isinstance(target, Pod):
+        prefix = ['oc', 'rsh', target.podname]
+        if isinstance(command, types.StringTypes):
+            cmd = ' '.join(prefix + [command])
+        else:
+            cmd = prefix + command
+
+        # unpack the tuple to make sure our return value exactly matches
+        # our docstring
+        return g.run(target.node, cmd, log_level=log_level)
+    else:
+        return orig_run(target, command, log_level=log_level)
+
+
+class GlustoPod(object):
+    """A context manager / decorator that monkeypatches the
+    glusto object to support running commands in pods.
+    """
+
+    def __init__(self, glusto_obj=None):
+        self.runfunc = None
+        self._g = glusto_obj or g
+
+    def __enter__(self):
+        """Patch glusto to use the wrapped run method.
+        """
+        self.runfunc = self._g.run
+        # we "capture" the prior glusto run method here in order to
+        # stack on top of any previous monkeypatches if they exist
+        self._g.run = partial(run, orig_run=self.runfunc)
+
+    def __exit__(self, etype, value, tb):
+        """Restore the orginal run method.
+        """
+        self._g.run = self.runfunc
+        self.runfunc = None
+
+    def __call__(self, func):
+        """Allow GlustoPod to be used as a decorator.
+        """
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with self:
+                result = func(*args, **kwargs)
+            return result
+        return wrapper
diff --git a/openshift-storage-libs/openshiftstoragelibs/utils.py b/openshift-storage-libs/openshiftstoragelibs/utils.py
new file mode 100644
index 00000000..2d16c497
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/utils.py
@@ -0,0 +1,40 @@
+"""Generic host utility functions.
+
+Generic utility functions not specifc to a larger suite of tools.
+For example, not specific to OCP, Gluster, Heketi, etc.
+"""
+
+import random
+import string
+
+from prometheus_client.parser import text_string_to_metric_families
+
+
+def get_random_str(size=14):
+    chars = string.ascii_lowercase + string.digits
+    return ''.join(random.choice(chars) for _ in range(size))
+
+
+def parse_prometheus_data(text):
+    """Parse prometheus-formatted text to the python objects
+
+    Args:
+        text (str): prometheus-formatted data
+
+    Returns:
+        dict: parsed data as python dictionary
+    """
+    metrics = {}
+    for family in text_string_to_metric_families(text):
+        for sample in family.samples:
+            key, data, val = (sample.name, sample.labels, sample.value)
+            if data.keys():
+                data['value'] = val
+                if key in metrics.keys():
+                    metrics[key].append(data)
+                else:
+                    metrics[key] = [data]
+            else:
+                metrics[key] = val
+
+    return metrics
diff --git a/openshift-storage-libs/openshiftstoragelibs/waiter.py b/openshift-storage-libs/openshiftstoragelibs/waiter.py
new file mode 100644
index 00000000..0d0c8c3a
--- /dev/null
+++ b/openshift-storage-libs/openshiftstoragelibs/waiter.py
@@ -0,0 +1,38 @@
+"""Helper object to encapsulate waiting for timeouts.
+
+Provide a Waiter class which encapsulates the operation
+of doing an action in a loop until a timeout values elapses.
+It aims to avoid having to write boilerplate code comparing times.
+"""
+
+import time
+
+
+class Waiter(object):
+    """A wait-retry loop as iterable.
+    This object abstracts away the wait logic allowing functions
+    to write the retry logic in a for-loop.
+    """
+    def __init__(self, timeout=60, interval=1):
+        self.timeout = timeout
+        self.interval = interval
+        self.expired = False
+        self._attempt = 0
+        self._start = None
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        if self._start is None:
+            self._start = time.time()
+        if time.time() - self._start > self.timeout:
+            self.expired = True
+            raise StopIteration()
+        if self._attempt != 0:
+            time.sleep(self.interval)
+        self._attempt += 1
+        return self
+
+    # NOTE(vponomar): py3 uses "__next__" method instead of "next" one.
+    __next__ = next
diff --git a/openshift-storage-libs/setup.py b/openshift-storage-libs/setup.py
new file mode 100644
index 00000000..3e528cbf
--- /dev/null
+++ b/openshift-storage-libs/setup.py
@@ -0,0 +1,30 @@
+#!/usr/bin/python
+from setuptools import setup, find_packages
+
+version = '0.2'
+name = 'openshift-storage-libs'
+
+setup(
+    name=name,
+    version=version,
+    description='Red Hat Container-Native Storage Libraries',
+    author='Red Hat, Inc.',
+    author_email='cns-qe@redhat.com',
+    packages=find_packages(),
+    include_package_data=True,
+    classifiers=[
+        'Development Status :: 3 - Alpha'
+        'Intended Audience :: QE, Developers'
+        'Operating System :: POSIX :: Linux'
+        'Programming Language :: Python'
+        'Programming Language :: Python :: 2'
+        'Programming Language :: Python :: 2.6'
+        'Programming Language :: Python :: 2.7'
+        'Topic :: Software Development :: Testing'
+    ],
+    install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff', 'six',
+                      'prometheus_client>=0.4.2'],
+    dependency_links=[
+        'http://github.com/loadtheaccumulator/glusto/tarball/master#egg=glusto'
+    ],
+)