summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArun Kumar <aanand01762@gmail.com>2020-03-19 16:16:26 +0530
committerVaibhav Mahajan <vamahaja@redhat.com>2020-04-28 14:10:30 +0000
commit28446571e6e36b73c76e546c2201f9859295e53b (patch)
tree98cdcfe64cedf816483530cd776e2528d5c2a7b3
parent549e96a856d4ec7d0258f7dfd1b0a50225504a7c (diff)
[Test] Add TCs to validate device/node disable based on heketi zones
Change-Id: I9786d8d68854529624a17cc2c2c12656b4eb6631 Signed-off-by: Arun Kumar <aanand01762@gmail.com>
-rw-r--r--tests/functional/heketi/test_heketi_zones.py326
1 files changed, 322 insertions, 4 deletions
diff --git a/tests/functional/heketi/test_heketi_zones.py b/tests/functional/heketi/test_heketi_zones.py
index 489092c..eb62b6b 100644
--- a/tests/functional/heketi/test_heketi_zones.py
+++ b/tests/functional/heketi/test_heketi_zones.py
@@ -5,15 +5,21 @@ try:
except ImportError:
# py2
import json
+from unittest import skip
import ddt
from glusto.core import Glusto as g
+import six
import pytest
from openshiftstoragelibs import baseclass
+from openshiftstoragelibs import command
+from openshiftstoragelibs import exceptions
from openshiftstoragelibs import heketi_ops
from openshiftstoragelibs import openshift_ops
from openshiftstoragelibs import openshift_storage_libs
+from openshiftstoragelibs import utils
+from openshiftstoragelibs.waiter import Waiter
@ddt.ddt
@@ -31,6 +37,8 @@ class TestHeketiZones(baseclass.BaseClass):
def setUp(self):
super(TestHeketiZones, self).setUp()
self.node = self.ocp_master_node[0]
+ self.h_client = self.heketi_client_node
+ self.h_server = self.heketi_server_url
def _set_heketi_zones(self, unique_zones_amount=1):
h = heketi_ops.cmd_run_on_heketi_pod
@@ -107,13 +115,11 @@ class TestHeketiZones(baseclass.BaseClass):
return heketi_db_data
def _get_online_nodes(self):
- node_ids = heketi_ops.heketi_node_list(
- self.heketi_client_node, self.heketi_server_url)
+ node_ids = heketi_ops.heketi_node_list(self.h_client, self.h_server)
online_nodes = []
for node_id in node_ids:
node_info = heketi_ops.heketi_node_info(
- self.heketi_client_node, self.heketi_server_url,
- node_id, json=True)
+ self.h_client, self.h_server, node_id, json=True)
if (node_info["state"] == "online"
and node_info['cluster'] == self.cluster_id):
online_nodes.append(
@@ -246,3 +252,315 @@ class TestHeketiZones(baseclass.BaseClass):
# Create app DC with the above PVC
self.create_dc_with_pvc(pvc_name, timeout=120, wait_step=3)
+
+ def _get_online_devices_and_nodes_with_zone(self):
+ """
+ This function returns the list of nodes and devices associated to zone
+
+ Returns:
+ dict: dict with zone, devices and nodes values
+ e.g.,
+ { zone_1: {
+ "nodes": [
+ node1, node2, node3],
+ "devices": [
+ device1, device2]
+ },
+ zone_2: {
+ "nodes": [
+ node1, node2, node3],
+ "devices": [
+ device1, device2]
+ }
+ }
+ """
+ zone_devices_nodes = dict()
+ topology_info = heketi_ops.heketi_topology_info(
+ self.h_client, self.h_server, json=True)
+ for cluster in topology_info['clusters']:
+ for node in cluster['nodes']:
+ if node['state'] == 'online':
+ if node['zone'] not in zone_devices_nodes:
+ zone_devices_nodes[node['zone']] = dict()
+
+ if 'nodes' not in zone_devices_nodes[node['zone']]:
+ zone_devices_nodes[node['zone']]['nodes'] = []
+
+ (zone_devices_nodes[node['zone']][
+ 'nodes'].append(node['id']))
+
+ for device in node['devices']:
+ if device['state'] == 'online':
+ if node['zone'] not in zone_devices_nodes:
+ zone_devices_nodes[node['zone']] = dict()
+
+ if 'devices' not in zone_devices_nodes[node['zone']]:
+ zone_devices_nodes[node['zone']]['devices'] = []
+
+ (zone_devices_nodes[
+ node['zone']]['devices'].append(device['id']))
+
+ return zone_devices_nodes
+
+ def _check_heketi_pod_to_come_up_after_changing_env(self):
+ err_str = 'Err: Error from server (NotFound): pods "{}" not found'
+ for w in Waiter(120, 5):
+ heketi_pod = openshift_ops.get_pod_names_from_dc(
+ self.node, self.heketi_dc_name)[0]
+ try:
+ openshift_ops.wait_for_pod_be_ready(
+ self.node, heketi_pod, 1, 1)
+ except (exceptions.ExecutionError, AssertionError) as e:
+ if err_str.format(heketi_pod) not in six.text_type(e):
+ raise
+ continue
+
+ break
+
+ if w.expired:
+ raise AssertionError(
+ "Heketi pod failing to come up after changing value of env "
+ "inside heketi dc")
+
+ def _set_zone_checking_option_in_heketi_dc_or_create_sc(
+ self, is_set_env, prefix):
+ sc_name = None
+ if is_set_env:
+ # Set env option for strict zone checking in heketi dc
+ set_env = ('HEKETI_POST_REQUEST_VOLUME_OPTIONS='
+ '"user.heketi.zone-checking strict"')
+ unset_env, e_list = "HEKETI_POST_REQUEST_VOLUME_OPTIONS-", "--list"
+ cmd_set_env = (
+ "oc set env dc/{} {}".format(self.heketi_dc_name, set_env))
+ cmd_unset_env = (
+ "oc set env dc/{} {}".format(self.heketi_dc_name, unset_env))
+ command.cmd_run(cmd_set_env, hostname=self.node)
+ self._check_heketi_pod_to_come_up_after_changing_env()
+ self.addCleanup(
+ self._check_heketi_pod_to_come_up_after_changing_env)
+ self.addCleanup(command.cmd_run, cmd_unset_env, hostname=self.node)
+
+ # List all envs and validate if env is set successfully
+ env = set_env.replace('"', '')
+ cmd_list_env = (
+ "oc set env dc/{} {}".format(self.heketi_dc_name, e_list))
+ env_list = command.cmd_run(cmd_list_env, hostname=self.node)
+ self.assertIn(env, env_list, "Failed to set env {}".format(env))
+
+ else:
+ # Create storage class setting "user.heketi.zone-checking" up
+ heketi_zone_checking = "strict"
+ sc_name = self.create_storage_class(
+ sc_name_prefix=prefix, vol_name_prefix=prefix,
+ heketi_zone_checking=heketi_zone_checking)
+
+ return sc_name
+
+ def _create_dcs_and_check_brick_placement(
+ self, prefix, sc_name, heketi_zone_checking, zone_count):
+ app_pods = []
+
+ # Create multiple PVCs using storage class
+ pvc_names = self.create_and_wait_for_pvcs(
+ pvc_name_prefix=prefix, pvc_amount=5, sc_name=sc_name)
+
+ # Create app dcs with I/O
+ for pvc_name in pvc_names:
+ app_dc = openshift_ops.oc_create_app_dc_with_io(
+ self.node, pvc_name=pvc_name, dc_name_prefix=prefix)
+ self.addCleanup(openshift_ops.oc_delete, self.node, 'dc', app_dc)
+
+ # Get pod names
+ pod_name = openshift_ops.get_pod_name_from_dc(self.node, app_dc)
+ app_pods.append(pod_name)
+
+ # Validate brick placement in heketi zones
+ self._validate_brick_placement_in_correct_zone_or_with_expand_pvc(
+ heketi_zone_checking, pvc_name, zone_count)
+
+ return app_pods
+
+ @skip("Blocked by BZ-1828249")
+ @pytest.mark.tier1
+ @ddt.data(
+ (3, False),
+ (3, True),
+ (4, True),
+ (3, False, True),
+ (3, True, True),
+ (4, True, True),
+ )
+ @ddt.unpack
+ def test_check_node_disable_based_on_heketi_zone(
+ self, zone_count, is_disable_on_different_zone, is_set_env=False):
+ """Validate node disable in different heketi zones"""
+ expected_node_count, heketi_zone_checking = 4, "strict"
+ prefix = "hzone-{}".format(utils.get_random_str())
+
+ # Check amount of available online nodes
+ online_node_count = len(self._get_online_nodes())
+ if online_node_count < expected_node_count:
+ self.skipTest(
+ 'Available node count {} is less than expected node '
+ 'count {}'.format(online_node_count, expected_node_count))
+
+ # Check amount of available online heketi zones
+ self._check_for_available_zones(zone_count)
+
+ # Get the online devices and nodes w.r.t. to zone
+ zone_devices_nodes = self._get_online_devices_and_nodes_with_zone()
+
+ # Set heketi zone checking option to "strict"
+ sc_name = self._set_zone_checking_option_in_heketi_dc_or_create_sc(
+ is_set_env, prefix)
+
+ # Choose a zone and node_id to disable the device
+ for zone, nodes_and_devices in zone_devices_nodes.items():
+ if zone_count == 3:
+ # Select a node with a zone having multiple nodes in same
+ # zone to cover the test cases disable node in same zone
+ if len(nodes_and_devices['nodes']) > 1:
+ zone_with_disabled_node = zone
+ disabled_node = nodes_and_devices['nodes'][0]
+ break
+
+ else:
+ # Select node from any of the zones
+ zone_with_disabled_node = zone
+ disabled_node = nodes_and_devices['nodes'][0]
+ break
+
+ # Disable the selected node
+ heketi_ops.heketi_node_disable(
+ self.h_client, self.h_server, disabled_node)
+ self.addCleanup(heketi_ops.heketi_node_enable, self.h_client,
+ self.h_server, disabled_node)
+
+ # Create some DCs with PVCs and check brick placement in heketi zones
+ pod_names = self._create_dcs_and_check_brick_placement(
+ prefix, sc_name, heketi_zone_checking, zone_count)
+
+ # Enable disabled node
+ heketi_ops.heketi_node_enable(
+ self.h_client, self.h_server, disabled_node)
+
+ if is_disable_on_different_zone:
+ # Select the new node in a different zone
+ for zone, nodes_and_devices in zone_devices_nodes.items():
+ if zone != zone_with_disabled_node:
+ new_node_to_disable = nodes_and_devices['nodes'][0]
+ break
+
+ else:
+ # Select the new node in the same zone
+ new_node_to_disable = zone_devices_nodes[
+ zone_with_disabled_node]['nodes'][1]
+
+ # Disable the newly selected node
+ heketi_ops.heketi_node_disable(
+ self.h_client, self.h_server, new_node_to_disable)
+ self.addCleanup(heketi_ops.heketi_node_enable, self.h_client,
+ self.h_server, new_node_to_disable)
+
+ # Verify if pods are in ready state
+ for pod_name in pod_names:
+ openshift_ops.wait_for_pod_be_ready(
+ self.node, pod_name, timeout=5, wait_step=2)
+
+ @skip("Blocked by BZ-1828249")
+ @pytest.mark.tier1
+ @ddt.data(
+ (3, False),
+ (3, True),
+ (4, True),
+ (3, False, True),
+ (3, True, True),
+ (4, True, True),
+ )
+ @ddt.unpack
+ def test_check_device_disable_based_on_heketi_zone(
+ self, zone_count, is_disable_on_different_zone, is_set_env=False):
+ """Validate device disable in different heketi zones"""
+ online_device_count, expected_device_count = 0, 4
+ expected_node_count, heketi_zone_checking = 4, "strict"
+ prefix = "hzone-{}".format(utils.get_random_str())
+
+ # Check amount of available online nodes
+ online_node_count = len(self._get_online_nodes())
+ if online_node_count < expected_node_count:
+ self.skipTest(
+ 'Available node count {} is less than expected node '
+ 'count {}'.format(online_node_count, expected_node_count))
+
+ # Check amount of available online heketi zones
+ self._check_for_available_zones(zone_count)
+
+ # Get the online devices and nodes w.r.t. to zone
+ zone_devices_nodes = self._get_online_devices_and_nodes_with_zone()
+
+ # Check amount of available online heketi devices
+ for zone in zone_devices_nodes:
+ online_device_count += len(
+ zone_devices_nodes[zone]['devices'])
+ if online_device_count < expected_device_count:
+ self.skipTest(
+ "Expected the heketi device count {} is greater than the "
+ "available device count {}".format(
+ expected_device_count, online_device_count))
+
+ # Set heketi zone checking option to "strict"
+ sc_name = self._set_zone_checking_option_in_heketi_dc_or_create_sc(
+ is_set_env, prefix)
+
+ # Choose a zone and device_id to disable the device
+ for zone, nodes_and_devices in zone_devices_nodes.items():
+ if zone_count == 3:
+ # Select a device with a zone having multiple nodes in
+ # same zone to cover the test cases "disable in same zone"
+ if len(nodes_and_devices['devices']) > 1:
+ zone_with_disabled_device = zone
+ disabled_device = nodes_and_devices['devices'][0]
+ break
+
+ else:
+ # Select device from any of the zones
+ zone_with_disabled_device = zone
+ disabled_device = nodes_and_devices['devices'][0]
+ break
+
+ # Disable the selected device
+ heketi_ops.heketi_device_disable(
+ self.h_client, self.h_server, disabled_device)
+ self.addCleanup(heketi_ops.heketi_device_enable, self.h_client,
+ self.h_server, disabled_device)
+
+ # Create some DCs with PVCs and check brick placement in heketi zones
+ pod_names = self._create_dcs_and_check_brick_placement(
+ prefix, sc_name, heketi_zone_checking, zone_count)
+
+ # Enable disabled device
+ heketi_ops.heketi_device_enable(
+ self.h_client, self.h_server, disabled_device)
+
+ if is_disable_on_different_zone:
+ # Select the new device in a different zone
+ for zone, nodes_and_devices in zone_devices_nodes.items():
+ if zone != zone_with_disabled_device:
+ new_device_to_disable = nodes_and_devices['devices'][0]
+ break
+
+ else:
+ # Select the new device in the same zone
+ new_device_to_disable = zone_devices_nodes[
+ zone_with_disabled_device]['devices'][1]
+
+ # Disable the newly selected device
+ heketi_ops.heketi_device_disable(
+ self.h_client, self.h_server, new_device_to_disable)
+ self.addCleanup(heketi_ops.heketi_device_enable, self.h_client,
+ self.h_server, new_device_to_disable)
+
+ # Verify if pods are in ready state
+ for pod_name in pod_names:
+ openshift_ops.wait_for_pod_be_ready(
+ self.node, pod_name, timeout=5, wait_step=2)