summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArun Kumar <arukumar@redhat.com>2020-09-09 17:04:45 +0530
committerVaibhav Mahajan <vamahaja@redhat.com>2020-09-28 16:50:33 +0000
commit4c8f312a8258818ed77d09a0f74aee35d1189290 (patch)
tree58f6443984160fe8f83a089797ecd476c9880308
parent5c38cc25d9f302f7717d9e3fe485096cb3d2871f (diff)
[Test] Add TC to perform post deployment validations of prometheus
Change-Id: I026f2aa944aaa6b338c45c54ac4504ecb854f00c Signed-off-by: Arun Kumar <arukumar@redhat.com>
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/openshift_ops.py35
-rw-r--r--tests/functional/prometheous/__init__.py0
-rw-r--r--tests/functional/prometheous/test_prometheus_validations.py116
-rw-r--r--tests/glusterfs-containers-tests-config.yaml7
4 files changed, 148 insertions, 10 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
index cf1e342..5f53dab 100644
--- a/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/openshift_ops.py
@@ -39,21 +39,24 @@ KILL_SERVICE = "kill -9 %s"
IS_ACTIVE_SERVICE = "systemctl is-active %s"
-def oc_get_pods(ocp_node, selector=None):
+def oc_get_pods(ocp_node, selector=None, name=None):
"""Gets the pods info with 'wide' option in the current project.
Args:
ocp_node (str): Node in which ocp command will be executed.
selector (str): optional option. Selector for OCP pods.
example: "glusterfs-node=pod" for filtering out only Gluster PODs.
+ name (str): name of the pod to get details.
Returns:
dict : dict of pods info in the current project.
"""
- cmd = "oc get -o wide --no-headers=true pods"
+ cmd = "oc get -o wide --no-headers=true pods "
if selector:
cmd += " --selector %s" % selector
+ if name:
+ cmd += name
out = command.cmd_run(cmd, hostname=ocp_node)
return _parse_wide_pods_output(out)
@@ -1100,13 +1103,15 @@ def wait_for_pod_be_ready(hostname, pod_name,
def wait_for_pods_be_ready(
- hostname, pod_count, selector, timeout=600, wait_step=10):
+ hostname, pod_count, selector=None, field_selector=None,
+ timeout=600, wait_step=10):
"""Wait to 'pod_count' gluster pods be in Ready state.
Args:
hostname (str): Node where we want to run our commands.
pod_count (int): No of pods to be waited for.
selector (str): Selector to select pods of given label.
+ field_selector (str): Selector to select pods.
timeout (int): Seconds to wait for Node to be Ready.
wait_step (int): Interval in seconds to wait before checking
status again.
@@ -1115,20 +1120,30 @@ def wait_for_pods_be_ready(
AssertionError: In case it fails to get pods.
ExecutionError: In case pods won't get in ready state for given time.
"""
- if not selector:
+ if not selector and not field_selector:
raise exceptions.ExecutionError(
- "selector parameter should be provided")
+ "Either selector or field-selector parameter should be provided")
custom = (
r':.metadata.name,":.status.conditions[?(@.type==\"Ready\")]".status')
pod_status = None
for w in waiter.Waiter(timeout, wait_step):
pod_status = oc_get_custom_resource(
- hostname, "pod", custom, selector=selector)
-
- if not pod_status:
- raise exceptions.ExecutionError(
- "Unable to find pod with selector %s" % selector)
+ hostname, "pod", custom, selector=selector,
+ field_selector=field_selector)
+
+ if not pod_status and pod_count != 0:
+ selection_text = ''
+ if selector and field_selector:
+ selection_text += 'selector {} and field-selector {}'.format(
+ selector, field_selector)
+ elif selector:
+ selection_text += 'selector {}'.format(selector)
+ else:
+ selection_text += 'field-selector {}'.format(field_selector)
+ raise exceptions.ExecutionError(
+ "Unable to find pods with mentioned {}".format(
+ selection_text))
status = [status for _, status in pod_status]
if len(status) == pod_count == status.count("True"):
return
diff --git a/tests/functional/prometheous/__init__.py b/tests/functional/prometheous/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/functional/prometheous/__init__.py
diff --git a/tests/functional/prometheous/test_prometheus_validations.py b/tests/functional/prometheous/test_prometheus_validations.py
new file mode 100644
index 0000000..6b296b5
--- /dev/null
+++ b/tests/functional/prometheous/test_prometheus_validations.py
@@ -0,0 +1,116 @@
+try:
+ # py2/3
+ import simplejson as json
+except ImportError:
+ # py2
+ import json
+from pkg_resources import parse_version
+
+from glusto.core import Glusto as g
+import pytest
+
+from openshiftstoragelibs.baseclass import GlusterBlockBaseClass
+from openshiftstoragelibs import command
+from openshiftstoragelibs import exceptions
+from openshiftstoragelibs import openshift_ops
+
+
+class TestPrometheusAndGlusterRegistryValidation(GlusterBlockBaseClass):
+
+ def setUp(self):
+ """Initialize all the variables which are necessary for test cases"""
+ super(TestPrometheusAndGlusterRegistryValidation, self).setUp()
+
+ try:
+ prometheus_config = g.config['openshift']['prometheus']
+ self._prometheus_project_name = prometheus_config[
+ 'prometheus_project_name']
+ self._prometheus_resources_selector = prometheus_config[
+ 'prometheus_resources_selector']
+ self._alertmanager_resources_selector = prometheus_config[
+ 'alertmanager_resources_selector']
+ self._registry_heketi_server_url = (
+ g.config['openshift']['registry_heketi_config'][
+ 'heketi_server_url'])
+ self._registry_project_name = (
+ g.config['openshift']['registry_project_name'])
+ except KeyError as err:
+ self.skipTest("Config file doesn't have key {}".format(err))
+
+ # Skip the test if iscsi-initiator-utils version is not the expected
+ cmd = ("rpm -q iscsi-initiator-utils "
+ "--queryformat '%{version}-%{release}\n'"
+ "| cut -d '.' -f 1,2,3,4")
+ e_pkg_version = "6.2.0.874-17"
+ for g_server in self.gluster_servers:
+ out = self.cmd_run(cmd, g_server)
+ if parse_version(out) < parse_version(e_pkg_version):
+ self.skipTest(
+ "Skip the test as iscsi-initiator-utils package version {}"
+ "is less than version {} found on the node {}, for more "
+ "info refer to BZ-1624670".format(
+ out, e_pkg_version, g_server))
+
+ self._master = self.ocp_master_node[0]
+
+ # Switch to namespace conatining prometheus pods
+ cmd = "oc project --short=true"
+ current_project = command.cmd_run(cmd, self._master)
+ openshift_ops.switch_oc_project(
+ self._master, self._prometheus_project_name)
+ self.addCleanup(
+ openshift_ops.switch_oc_project, self._master, current_project)
+
+ def _fetch_metric_from_promtheus_pod(self, metric):
+ """Fetch metric from prometheus pod using api call"""
+ prometheus_pods = list(openshift_ops.oc_get_pods(
+ self._master, selector=self._prometheus_resources_selector).keys())
+ fetch_metric_cmd = ("curl 'http://localhost:9090/api/v1/query"
+ "?query={}'".format(metric))
+ ret, metric_data, _ = openshift_ops.oc_rsh(
+ self._master, prometheus_pods[0], fetch_metric_cmd)
+ metric_result = json.loads(metric_data)["data"]["result"]
+ if (not metric_result) or ret:
+ raise exceptions.ExecutionError(
+ "Failed to fecth data for metric {}, output {}".format(
+ metric, metric_result))
+ return metric_result
+
+ def _get_pod_names_and_pvc_names(self):
+ # Get pod names and PVC names
+ pod_custom = ".:metadata.name"
+ pvc_custom = ":.spec.volumes[*].persistentVolumeClaim.claimName"
+ pvc_names, pod_names = [], []
+ for selector in (self._prometheus_resources_selector,
+ self._alertmanager_resources_selector):
+ pods = openshift_ops.oc_get_custom_resource(
+ self._master, "pod", pod_custom, selector=selector)
+ pod_names.extend(pods)
+ for pod_name in pods:
+ pvc_name = openshift_ops.oc_get_custom_resource(
+ self._master, "pod", pvc_custom, pod_name[0])[0]
+ pvc_names.append(pvc_name)
+
+ return pod_names, pvc_names
+
+ @pytest.mark.tier2
+ def test_promethoues_pods_and_pvcs(self):
+ """Validate prometheus pods and PVC"""
+ # Wait for PVCs to be bound
+ pod_names, pvc_names = self._get_pod_names_and_pvc_names()
+ openshift_ops.wait_for_pvcs_be_bound(self._master, pvc_names)
+
+ # Validate that there should be no or zero pods in non-running state
+ field_selector, pod_count = "status.phase!=Running", 0
+ openshift_ops.wait_for_pods_be_ready(
+ self._master, pod_count, field_selector=field_selector)
+
+ # Validate iscsi and multipath
+ for (pvc_name, pod_name) in zip(pvc_names, pod_names):
+ self.verify_iscsi_sessions_and_multipath(
+ pvc_name, pod_name[0], rtype='pod',
+ heketi_server_url=self._registry_heketi_server_url,
+ is_registry_gluster=True)
+
+ # Try to fetch metric from prometheus pod
+ self._fetch_metric_from_promtheus_pod(metric='kube_node_info')
diff --git a/tests/glusterfs-containers-tests-config.yaml b/tests/glusterfs-containers-tests-config.yaml
index 4b73cf5..65409c4 100644
--- a/tests/glusterfs-containers-tests-config.yaml
+++ b/tests/glusterfs-containers-tests-config.yaml
@@ -107,6 +107,13 @@ openshift:
logging_es_dc: "<elasticsearch-dc-name"
logging_kibana_dc: "<kibana-dc-name>"
+ # 'prometheus' section covers the details of resources related to
+ # prometheus
+ prometheus:
+ prometheus_project_name: "<prometheus_project_name>"
+ prometheus_resources_selector: "<prometheus_recources_selector>"
+ alertmanager_resources_selector: "<alertmanager_resources_selector>"
+
common:
allow_heketi_zones_update: False
check_heketi_db_inconsistencies: True