Fix 'verify_arbiter_brick_able_to_contain_expected_amount_of_files' tc

This test case talks to the Gluster PODs directly. It is not
compatible with Standalone Gluster clusters.
So, fix this test by reusing special common function,
which allows us to abstract from Gluster cluster deployment type.

Change-Id: Ib131984ac0a4623d1d847f9961be26da573d8513

2 files changed, 59 insertions, 30 deletions

diff --git a/cns-libs/cnslibs/common/openshift_ops.py b/cns-libs/cnslibs/common/openshift_ops.py
index 9a84ee83..d98c550b 100644
--- a/cns-libs/cnslibs/common/openshift_ops.py
+++ b/cns-libs/cnslibs/common/openshift_ops.py
@@ -30,17 +30,21 @@ SERVICE_RESTART = "systemctl restart %s"
 SERVICE_STATUS_REGEX = r"Active: active \((.*)\) since .*;.*"
 
 
-def oc_get_pods(ocp_node):
+def oc_get_pods(ocp_node, selector=None):
     """Gets the pods info with 'wide' option in the current project.
 
     Args:
         ocp_node (str): Node in which ocp command will be executed.
+        selector (str): optional option. Selector for OCP pods.
+            example: "glusterfs-node=pod" for filtering out only Gluster PODs.
 
     Returns:
         dict : dict of pods info in the current project.
     """
 
     cmd = "oc get -o wide --no-headers=true pods"
+    if selector:
+        cmd += " --selector %s" % selector
     ret, out, err = g.run(ocp_node, cmd)
     if ret != 0:
         g.log.error("Failed to get ocp pods on node %s" % ocp_node)
@@ -772,23 +776,51 @@ def get_gluster_pod_names_by_pvc_name(ocp_node, pvc_name):
     return data
 
 
-def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd):
-    """Run shell command on either Gluster POD or Gluster node.
+def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd, gluster_node=None):
+    """Run shell command on either Gluster PODs or Gluster nodes.
 
     Args:
         ocp_client_node (str): Node to execute OCP commands on.
         cmd (str): shell command to run.
+        gluster_node (str): optional. Allows to chose specific gluster node,
+            keeping abstraction from deployment type. Can be either IP address
+            or node name from "oc get nodes" command.
     Returns:
         Output of a shell command as string object.
     """
     # Containerized Glusterfs
-    gluster_pods = get_ocp_gluster_pod_names(ocp_client_node)
+    gluster_pods = oc_get_pods(ocp_client_node, selector="glusterfs-node=pod")
     if gluster_pods:
-        pod_cmd = "oc exec %s -- %s" % (gluster_pods[0], cmd)
-        return command.cmd_run(pod_cmd, hostname=ocp_client_node)
+        if gluster_node:
+            for pod_name, pod_data in gluster_pods.items():
+                if gluster_node in (pod_data["ip"], pod_data["node"]):
+                    gluster_pod_names = [pod_name]
+                    break
+            else:
+                raise exceptions.ExecutionError(
+                    "Could not find Gluster PODs with node filter as "
+                    "'%s'." % gluster_node)
+        else:
+            gluster_pod_names = gluster_pods.keys()
+
+        err_msg = ""
+        for gluster_pod_name in gluster_pod_names:
+            try:
+                pod_cmd = "oc exec %s -- %s" % (gluster_pod_name, cmd)
+                return command.cmd_run(pod_cmd, hostname=ocp_client_node)
+            except Exception as e:
+                err = ("Failed to run '%s' command on '%s' Gluster POD. "
+                       "Error: %s\n" % (cmd, gluster_pod_name, e))
+                err_msg += err
+                g.log.error(err)
+        raise exceptions.ExecutionError(err_msg)
 
     # Standalone Glusterfs
-    for g_host in g.config.get("gluster_servers", {}).keys():
+    if gluster_node:
+        g_hosts = [gluster_node]
+    else:
+        g_hosts = g.config.get("gluster_servers", {}).keys()
+    for g_host in g_hosts:
         try:
             return command.cmd_run(cmd, hostname=g_host)
         except Exception as e:
@@ -796,7 +828,7 @@ def cmd_run_on_gluster_pod_or_node(ocp_client_node, cmd):
                 "Failed to run '%s' command on '%s' Gluster node. "
                 "Error: %s" % (cmd, g_host,  e))
 
-    raise exceptions.ConfigError(
+    raise exceptions.ExecutionError(
         "Haven't found neither Gluster PODs nor Gluster nodes.")
 
 
diff --git a/tests/functional/common/arbiter/test_arbiter.py b/tests/functional/common/arbiter/test_arbiter.py
index 4a0853b3..d4b391f6 100644
--- a/tests/functional/common/arbiter/test_arbiter.py
+++ b/tests/functional/common/arbiter/test_arbiter.py
@@ -3,8 +3,8 @@ import ddt
 from cnslibs.cns import cns_baseclass
 from cnslibs.common import heketi_ops
 from cnslibs.common.openshift_ops import (
+    cmd_run_on_gluster_pod_or_node,
     get_gluster_vol_info_by_pvc_name,
-    get_ocp_gluster_pod_names,
     oc_create_pvc,
     oc_create_tiny_pod_with_volume,
     oc_delete,
@@ -346,33 +346,30 @@ class TestArbiterVolumeCreateExpandDelete(cns_baseclass.BaseClass):
         # Try to create expected amount of files on arbiter brick mount
         passed_arbiter_bricks = []
         not_found = "Mount Not Found"
-        gluster_pods = get_ocp_gluster_pod_names(self.node)
         for brick in bricks_info['arbiter_list']:
-            for gluster_pod in gluster_pods:
-                # "brick path" looks like following:
-                # ip_addr:/path/to/vg/brick_unique_name/brick
-                # So, we remove "ip_addr" and "/brick" parts to have mount path
-                brick_path = brick["name"].split(":")[-1]
-                cmd = "oc exec %s -- mount | grep %s || echo '%s'" % (
-                    gluster_pod, brick_path[0:-6], not_found)
-                out = self.cmd_run(cmd)
-                if out != not_found:
-                    cmd = (
-                        "oc exec %s -- python -c \"["
-                        "    open('%s/foo_file{0}'.format(i), 'a').close()"
-                        "    for i in range(%s)"
-                        "]\"" % (gluster_pod, brick_path, expected_file_amount)
-                    )
-                    out = self.cmd_run(cmd)
-                    passed_arbiter_bricks.append(brick_path)
-                    break
+            # "brick path" looks like following:
+            # ip_addr:/path/to/vg/brick_unique_name/brick
+            gluster_ip, brick_path = brick["name"].split(":")
+            brick_path = brick_path[0:-6]
+
+            cmd = "mount | grep %s || echo '%s'" % (brick_path, not_found)
+            out = cmd_run_on_gluster_pod_or_node(self.node, cmd, gluster_ip)
+            if out != not_found:
+                cmd = (
+                    "python -c \"["
+                    "    open('%s/foo_file{0}'.format(i), 'a').close()"
+                    "    for i in range(%s)"
+                    "]\"" % (brick_path, expected_file_amount)
+                )
+                cmd_run_on_gluster_pod_or_node(self.node, cmd, gluster_ip)
+                passed_arbiter_bricks.append(brick["name"])
 
         # Make sure all the arbiter bricks were checked
         for brick in bricks_info['arbiter_list']:
             self.assertIn(
-                brick["name"].split(":")[-1], passed_arbiter_bricks,
+                brick["name"], passed_arbiter_bricks,
                 "Arbiter brick '%s' was not verified. Looks like it was "
-                "not found on any of gluster nodes." % brick_path)
+                "not found on any of gluster PODs/nodes." % brick["name"])
 
     @ddt.data(True, False)
     def test_aribiter_required_tag_on_node_or_devices_other_disabled(