diff options
| -rw-r--r-- | cns-libs/cnslibs/common/heketi_ops.py | 29 | ||||
| -rw-r--r-- | cns-libs/cnslibs/common/utils.py | 26 | ||||
| -rw-r--r-- | cns-libs/setup.py | 3 | ||||
| -rw-r--r-- | tests/functional/common/heketi/test_heketi_metrics.py | 272 | ||||
| -rw-r--r-- | tox.ini | 1 |
5 files changed, 330 insertions, 1 deletions
diff --git a/cns-libs/cnslibs/common/heketi_ops.py b/cns-libs/cnslibs/common/heketi_ops.py index fea574da..534017ff 100644 --- a/cns-libs/cnslibs/common/heketi_ops.py +++ b/cns-libs/cnslibs/common/heketi_ops.py @@ -15,6 +15,7 @@ except ImportError: g.log.error("Please install python-client for heketi and re-run the test") from cnslibs.common import exceptions, podcmd +from cnslibs.common.utils import parse_prometheus_data HEKETI_SSH_KEY = "/etc/heketi/heketi_key" HEKETI_CONFIG_FILE = "/etc/heketi/heketi.json" @@ -2388,3 +2389,31 @@ def match_heketi_and_gluster_block_volumes( assert sorted(gluster_vol_block_list) == heketi_block_volumes, ( "Gluster and Heketi Block volume list match failed") + + +def get_heketi_metrics(heketi_client_node, heketi_server_url, + prometheus_format=False): + ''' Execute curl command to get metrics output + + Args: + - heketi_client_node (str) : Node where we want to run our commands. + - heketi_server_url (str) : This is a heketi server url + - prometheus_format (bool) : control the format of output + by default it is False, So it will parse prometheus format into + python dict. If we need prometheus format we have to set it True. + Returns: + Metrics output: if successful + Raises: + err: if fails to run command + + ''' + + cmd = "curl %s/metrics" % heketi_server_url + ret, out, err = g.run(heketi_client_node, cmd) + if ret != 0: + msg = "failed to get Heketi metrics with following error: %s" % err + g.log.error(msg) + raise AssertionError(msg) + if prometheus_format: + return out.strip() + return parse_prometheus_data(out) diff --git a/cns-libs/cnslibs/common/utils.py b/cns-libs/cnslibs/common/utils.py index 7d1f6d6f..9aa38ff9 100644 --- a/cns-libs/cnslibs/common/utils.py +++ b/cns-libs/cnslibs/common/utils.py @@ -10,6 +10,7 @@ import string from glusto.core import Glusto as g +from prometheus_client.parser import text_string_to_metric_families ONE_GB_BYTES = 1073741824.0 @@ -49,3 +50,28 @@ def get_device_size(host, device_name): def get_random_str(size=14): chars = string.ascii_lowercase + string.digits return ''.join(random.choice(chars) for _ in range(size)) + + +def parse_prometheus_data(text): + """Parse prometheus-formatted text to the python objects + + Args: + text (str): prometheus-formatted data + + Returns: + dict: parsed data as python dictionary + """ + metrics = {} + for family in text_string_to_metric_families(text): + for sample in family.samples: + key, data, val = (sample.name, sample.labels, sample.value) + if data.keys(): + data['value'] = val + if key in metrics.keys(): + metrics[key].append(data) + else: + metrics[key] = [data] + else: + metrics[key] = val + + return metrics diff --git a/cns-libs/setup.py b/cns-libs/setup.py index 42f1fd83..bb3803a9 100644 --- a/cns-libs/setup.py +++ b/cns-libs/setup.py @@ -22,7 +22,8 @@ setup( 'Programming Language :: Python :: 2.7' 'Topic :: Software Development :: Testing' ], - install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff', 'six'], + install_requires=['glusto', 'ddt', 'mock', 'rtyaml', 'jsondiff', 'six', + 'prometheus_client>=0.4.2'], dependency_links=[ 'http://github.com/loadtheaccumulator/glusto/tarball/master#egg=glusto' ], diff --git a/tests/functional/common/heketi/test_heketi_metrics.py b/tests/functional/common/heketi/test_heketi_metrics.py new file mode 100644 index 00000000..04147e37 --- /dev/null +++ b/tests/functional/common/heketi/test_heketi_metrics.py @@ -0,0 +1,272 @@ +from cnslibs.common.heketi_libs import HeketiClientSetupBaseClass +from cnslibs.common.heketi_ops import ( + get_heketi_metrics, + heketi_cluster_info, + heketi_topology_info, + heketi_volume_create, + heketi_volume_delete, + heketi_volume_list + ) +from cnslibs.common.openshift_ops import ( + get_pod_name_from_dc, + scale_dc_pod_amount_and_wait, + wait_for_pod_be_ready + ) + + +class TestHeketiMetrics(HeketiClientSetupBaseClass): + + def verify_heketi_metrics_with_topology_info(self): + topology = heketi_topology_info( + self.heketi_client_node, self.heketi_server_url, json=True) + + metrics = get_heketi_metrics( + self.heketi_client_node, self.heketi_server_url) + + self.assertTrue(topology) + self.assertIn('clusters', list(topology.keys())) + self.assertGreater(len(topology['clusters']), 0) + + self.assertTrue(metrics) + self.assertGreater(len(metrics.keys()), 0) + + self.assertEqual( + len(topology['clusters']), metrics['heketi_cluster_count']) + + for cluster in topology['clusters']: + self.assertIn('nodes', list(cluster.keys())) + self.assertGreater(len(cluster['nodes']), 0) + + cluster_id = cluster['id'] + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_nodes_count']]) + self.assertIn(cluster_id, cluster_ids) + for node_count in metrics['heketi_nodes_count']: + if node_count['cluster'] == cluster_id: + self.assertEqual( + len(cluster['nodes']), node_count['value']) + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_volumes_count']]) + self.assertIn(cluster_id, cluster_ids) + for vol_count in metrics['heketi_volumes_count']: + if vol_count['cluster'] == cluster_id: + self.assertEqual( + len(cluster['volumes']), vol_count['value']) + + for node in cluster['nodes']: + self.assertIn('devices', list(node.keys())) + self.assertGreater(len(node['devices']), 0) + + hostname = node['hostnames']['manage'][0] + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_device_count']]) + self.assertIn(cluster_id, cluster_ids) + hostnames = ([obj['hostname'] + for obj in metrics['heketi_device_count']]) + self.assertIn(hostname, hostnames) + for device_count in metrics['heketi_device_count']: + if (device_count['cluster'] == cluster_id and + device_count['hostname'] == hostname): + self.assertEqual( + len(node['devices']), device_count['value']) + + for device in node['devices']: + device_name = device['name'] + device_size_t = device['storage']['total'] + device_free_t = device['storage']['free'] + device_used_t = device['storage']['used'] + + cluster_ids = ([obj['cluster'] + for obj in + metrics['heketi_device_brick_count']]) + self.assertIn(cluster_id, cluster_ids) + hostnames = ([obj['hostname'] + for obj in + metrics['heketi_device_brick_count']]) + self.assertIn(hostname, hostnames) + devices = ([obj['device'] + for obj in + metrics['heketi_device_brick_count']]) + self.assertIn(device_name, devices) + for brick_count in metrics['heketi_device_brick_count']: + if (brick_count['cluster'] == cluster_id and + brick_count['hostname'] == hostname and + brick_count['device'] == device_name): + self.assertEqual( + len(device['bricks']), brick_count['value']) + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_device_size']]) + self.assertIn(cluster_id, cluster_ids) + hostnames = ([obj['hostname'] + for obj in metrics['heketi_device_size']]) + self.assertIn(hostname, hostnames) + devices = ([obj['device'] + for obj in metrics['heketi_device_size']]) + self.assertIn(device_name, devices) + for device_size in metrics['heketi_device_size']: + if (device_size['cluster'] == cluster_id and + device_size['hostname'] == hostname and + device_size['device'] == device_name): + self.assertEqual( + device_size_t, device_size['value']) + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_device_free']]) + self.assertIn(cluster_id, cluster_ids) + hostnames = ([obj['hostname'] + for obj in metrics['heketi_device_free']]) + self.assertIn(hostname, hostnames) + devices = ([obj['device'] + for obj in metrics['heketi_device_free']]) + self.assertIn(device_name, devices) + for device_free in metrics['heketi_device_free']: + if (device_free['cluster'] == cluster_id and + device_free['hostname'] == hostname and + device_free['device'] == device_name): + self.assertEqual( + device_free_t, device_free['value']) + + cluster_ids = ([obj['cluster'] + for obj in metrics['heketi_device_used']]) + self.assertIn(cluster_id, cluster_ids) + hostnames = ([obj['hostname'] + for obj in metrics['heketi_device_used']]) + self.assertIn(hostname, hostnames) + devices = ([obj['device'] + for obj in metrics['heketi_device_used']]) + self.assertIn(device_name, devices) + for device_used in metrics['heketi_device_used']: + if (device_used['cluster'] == cluster_id and + device_used['hostname'] == hostname and + device_used['device'] == device_name): + self.assertEqual( + device_used_t, device_used['value']) + + def verify_volume_count(self): + metrics = get_heketi_metrics( + self.heketi_client_node, + self.heketi_server_url) + self.assertTrue(metrics['heketi_volumes_count']) + + for vol_count in metrics['heketi_volumes_count']: + self.assertTrue(vol_count['cluster']) + cluster_info = heketi_cluster_info( + self.heketi_client_node, + self.heketi_server_url, + vol_count['cluster'], json=True) + self.assertEqual(vol_count['value'], len(cluster_info['volumes'])) + + def test_heketi_metrics_with_topology_info(self): + # CNS-1243 - Heketi_metrics_generate + self.verify_heketi_metrics_with_topology_info() + + def test_heketi_metrics_heketipod_failure(self): + # CNS-1262 - Heketi-metrics_validating_heketi_pod failure + scale_dc_pod_amount_and_wait( + self.ocp_master_node, self.heketi_dc_name, pod_amount=0) + self.addCleanup( + scale_dc_pod_amount_and_wait, self.ocp_master_node, + self.heketi_dc_name, pod_amount=1) + + # verify that metrics is not accessable when heketi pod is down + with self.assertRaises(AssertionError): + get_heketi_metrics( + self.heketi_client_node, + self.heketi_server_url, + prometheus_format=True) + + scale_dc_pod_amount_and_wait( + self.ocp_master_node, self.heketi_dc_name, pod_amount=1) + + pod_name = get_pod_name_from_dc( + self.ocp_master_node, self.heketi_dc_name, self.heketi_dc_name) + wait_for_pod_be_ready(self.ocp_master_node, pod_name, wait_step=5) + + for i in range(3): + vol = heketi_volume_create( + self.heketi_client_node, + self.heketi_server_url, 1, json=True) + + self.assertTrue(vol) + + self.addCleanup( + heketi_volume_delete, + self.heketi_client_node, + self.heketi_server_url, + vol['id'], + raise_on_error=False) + + vol_list = heketi_volume_list( + self.heketi_client_node, + self.heketi_server_url) + + self.assertIn(vol['id'], vol_list) + + self.verify_heketi_metrics_with_topology_info() + + def test_heketi_metrics_validating_vol_count_on_vol_creation(self): + # CNS-1244 - Heketi_metrics_validating_VolumeCount_on_creation + + for i in range(3): + # Create volume + vol = heketi_volume_create( + self.heketi_client_node, + self.heketi_server_url, 1, json=True) + self.assertTrue(vol) + self.addCleanup( + heketi_volume_delete, + self.heketi_client_node, + self.heketi_server_url, + vol['id'], + raise_on_error=False) + + vol_list = heketi_volume_list( + self.heketi_client_node, + self.heketi_server_url) + + self.assertIn(vol['id'], vol_list) + + self.verify_volume_count() + + def test_heketi_metrics_validating_vol_count_on_vol_deletion(self): + # CNS-1245 - Heketi_metrics_validating_VolumeCount_on_deletion + + vol_list = [] + + for i in range(3): + # Create volume + vol = heketi_volume_create( + self.heketi_client_node, + self.heketi_server_url, 1, json=True) + + self.assertTrue(vol) + + self.addCleanup( + heketi_volume_delete, + self.heketi_client_node, + self.heketi_server_url, + vol['id'], + raise_on_error=False) + + volume_list = heketi_volume_list( + self.heketi_client_node, + self.heketi_server_url) + + self.assertIn(vol['id'], volume_list) + vol_list.append(vol) + + for vol in vol_list: + # delete volume + heketi_volume_delete( + self.heketi_client_node, + self.heketi_server_url, + vol['id']) + volume_list = heketi_volume_list( + self.heketi_client_node, + self.heketi_server_url) + self.assertNotIn(vol['id'], volume_list) + self.verify_volume_count() @@ -24,6 +24,7 @@ commands = mock \ rtyaml \ ddt \ + prometheus_client>=0.4.2 \ git+git://github.com/loadtheaccumulator/glusto.git \ "git+git://github.com/gluster/glusto-tests.git#egg=glustolibs-gluster&subdirectory=glustolibs-gluster" \ "git+git://github.com/gluster/glusto-tests.git#egg=glustolibs-io&subdirectory=glustolibs-io" \ |