summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/gluster_ops.py2
-rw-r--r--openshift-storage-libs/openshiftstoragelibs/heketi_ops.py97
-rw-r--r--tests/functional/gluster_stability/test_restart_gluster_services.py29
-rw-r--r--tests/functional/provisioning/test_storage_class_cases.py15
4 files changed, 136 insertions, 7 deletions
diff --git a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
index 798e3c2..950fd07 100644
--- a/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/gluster_ops.py
@@ -96,7 +96,7 @@ def restart_gluster_vol_brick_processes(ocp_client_node, file_vol,
# Get Gluster vol brick PIDs
gluster_volume_status = get_gluster_vol_status(file_vol)
- pids = ()
+ pids = []
for gluster_node in gluster_nodes:
pid = None
for g_node, g_node_data in gluster_volume_status.items():
diff --git a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
index 0021e79..78c2e07 100644
--- a/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
+++ b/openshift-storage-libs/openshiftstoragelibs/heketi_ops.py
@@ -12,10 +12,14 @@ from glusto.core import Glusto as g
import six
from openshiftstoragelibs import command
+from openshiftstoragelibs import exceptions
from openshiftstoragelibs import heketi_version
from openshiftstoragelibs.utils import parse_prometheus_data
+from openshiftstoragelibs import waiter
+
HEKETI_BHV = re.compile(r"Id:(\S+)\s+Cluster:(\S+)\s+Name:(\S+)\s\[block\]")
+HEKETI_OPERATIONS = re.compile(r"Id:(\S+)\s+Type:(\S+)\s+Status:(\S+)")
HEKETI_COMMAND_TIMEOUT = g.config.get("common", {}).get(
"heketi_command_timeout", 120)
TIMEOUT_PREFIX = "timeout %s " % HEKETI_COMMAND_TIMEOUT
@@ -1446,3 +1450,96 @@ def get_total_free_space(heketi_client_node, heketi_server_url):
total_device_free_space += (device["storage"]["free"])
device_free_spaces.append(total_device_free_space / 1024 ** 2)
return int(sum(device_free_spaces)), len(device_free_spaces)
+
+
+def heketi_server_operations_list(
+ heketi_client_node, heketi_server_url, **kwargs):
+ """Executes heketi server operations list command.
+
+ Args:
+ heketi_client_node (str): Node on which cmd has to be executed.
+ heketi_server_url (str): Heketi server url
+
+ Returns:
+ list: list of server operations pending
+
+ Raises:
+ exceptions.ExecutionError: if command fails.
+ """
+ version = heketi_version.get_heketi_version(heketi_client_node)
+ if version < '8.0.0-10':
+ msg = (
+ "heketi-client package %s does not support operations "
+ "list functionality" % version.v_str)
+ g.log.error(msg)
+ raise NotImplementedError(msg)
+
+ heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+ heketi_server_url, **kwargs)
+
+ cmd = "heketi-cli -s %s %s %s server operations list" % (
+ heketi_server_url, admin_key, user)
+ cmd = TIMEOUT_PREFIX + cmd
+ out = heketi_cmd_run(heketi_client_node, cmd)
+ if out:
+ operations = []
+ for operation in HEKETI_OPERATIONS.findall(out.strip()):
+ operations.append({
+ 'id': operation[0],
+ 'type': operation[1],
+ 'status': operation[2]
+ })
+ return operations
+ else:
+ g.log.info("No any pendig heketi server operation")
+ return []
+
+
+def heketi_server_operation_cleanup(
+ heketi_client_node, heketi_server_url, operation_id=None,
+ timeout=120, wait_time=5, **kwargs):
+ """Executes heketi server operations cleanup command and wait until
+ cleanup operations get completed for given timeout.
+
+ Args:
+ heketi_client_node (str): Node on which cmd has to be executed.
+ heketi_server_url (str): Heketi server url
+ operation_id (str): Operation Id needs to be cleaned.
+
+ Raises:
+ exceptions.ExecutionError: If cleanup not completed in given timeout.
+ """
+ version = heketi_version.get_heketi_version(heketi_client_node)
+ if version < '8.0.0-10':
+ msg = (
+ "heketi-client package %s does not support operations "
+ "cleanup functionality" % version.v_str)
+ g.log.error(msg)
+ raise NotImplementedError(msg)
+
+ heketi_server_url, json_arg, admin_key, user = _set_heketi_global_flags(
+ heketi_server_url, **kwargs)
+ cmd = "heketi-cli -s %s %s %s server operations cleanup" % (
+ heketi_server_url, admin_key, user)
+ if operation_id:
+ cmd += " %s" % operation_id
+
+ cmd = TIMEOUT_PREFIX + cmd
+ heketi_cmd_run(heketi_client_node, cmd)
+ for w in waiter.Waiter(timeout=timeout, interval=wait_time):
+ cleanup_operations = heketi_server_operations_list(
+ heketi_client_node, heketi_server_url, **kwargs)
+
+ cleanup_operation = [
+ operation["id"]
+ for operation in cleanup_operations
+ if operation["id"] == operation_id]
+ if not cleanup_operation:
+ break
+
+ if w.expired:
+ err_msg = (
+ "Heketi server cleanup operation still pending even "
+ "after %s second" % timeout)
+ g.log.error(err_msg)
+ raise exceptions.ExecutionError(err_msg)
diff --git a/tests/functional/gluster_stability/test_restart_gluster_services.py b/tests/functional/gluster_stability/test_restart_gluster_services.py
index b68da6b..8506968 100644
--- a/tests/functional/gluster_stability/test_restart_gluster_services.py
+++ b/tests/functional/gluster_stability/test_restart_gluster_services.py
@@ -15,7 +15,11 @@ from openshiftstoragelibs.gluster_ops import (
restart_gluster_vol_brick_processes,
wait_to_heal_complete,
)
-from openshiftstoragelibs.heketi_ops import heketi_blockvolume_list
+from openshiftstoragelibs.heketi_ops import (
+ heketi_blockvolume_list,
+ heketi_server_operation_cleanup,
+ heketi_server_operations_list,
+)
from openshiftstoragelibs.openshift_ops import (
get_pv_name_from_pvc,
match_pv_and_heketi_block_volumes,
@@ -139,6 +143,17 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass):
def validate_volumes_and_blocks(self):
"""Validates PVC and block volumes generated through heketi and OCS
"""
+ heketi_operations = heketi_server_operations_list(
+ self.heketi_client_node, self.heketi_server_url,
+ secret=self.heketi_cli_key, user=self.heketi_cli_user)
+
+ for heketi_operation in heketi_operations:
+ if heketi_operation["status"] == "failed":
+ heketi_server_operation_cleanup(
+ self.heketi_client_node, self.heketi_server_url,
+ heketi_operation["id"], secret=self.heketi_cli_key,
+ user=self.heketi_cli_user
+ )
# verify pvc status is in "Bound" for all the pvc
for pvc in self.pvc_list:
@@ -204,9 +219,21 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass):
wait_to_heal_complete()
+ @skip("Blocked by BZ-1634745, BZ-1635736, BZ-1636477")
@ddt.data(SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET)
def test_restart_services_provision_volume_and_run_io(self, service):
"""Restart gluster service then validate volumes"""
+ skip_msg = (
+ "Skipping this test case due to bugs "
+ "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668")
+
+ # TODO(vamahaja): Add check for CRS version
+ if not self.is_containerized_gluster():
+ self.skipTest(skip_msg + " and not implemented CRS version check")
+
+ if get_openshift_storage_version() < "3.11.2":
+ self.skipTest(skip_msg)
+
self.deploy_and_verify_resouces()
block_hosting_vol = self.get_block_hosting_volume_by_pvc_name(
diff --git a/tests/functional/provisioning/test_storage_class_cases.py b/tests/functional/provisioning/test_storage_class_cases.py
index 6cf6e96..2e319f6 100644
--- a/tests/functional/provisioning/test_storage_class_cases.py
+++ b/tests/functional/provisioning/test_storage_class_cases.py
@@ -1,5 +1,3 @@
-from unittest import skip
-
import ddt
from glusto.core import Glusto as g
@@ -261,7 +259,6 @@ class TestStorageClassCases(BaseClass):
)
self.validate_multipath_info(hacount)
- @skip("Blocked by BZ-1644685")
def test_gluster_block_provisioning_with_invalid_ha_count(self):
"""Validate gluster-block provisioning with any invalid 'hacount'
value
@@ -278,12 +275,20 @@ class TestStorageClassCases(BaseClass):
"is not supported in OCS 3.9")
# get hacount as no of gluster pods + 1 to fail the pvc creation
- hacount = get_amount_of_gluster_nodes(self.ocp_master_node[0]) + 1
+ gluster_pod_count = get_amount_of_gluster_nodes(
+ self.ocp_master_node[0])
+ hacount = gluster_pod_count + 1
# create storage class and pvc with given parameters
self.create_sc_with_parameter(
- 'glusterblock', parameter={'hacount': str(hacount)}
+ 'glusterblock', success=True, parameter={'hacount': str(hacount)}
+ )
+
+ # validate HA parameter with gluster block volume
+ self.validate_gluster_block_volume_info(
+ self.assertEqual, 'HA', gluster_pod_count
)
+ self.validate_multipath_info(gluster_pod_count)
@ddt.data('true', 'false', '')
def test_gluster_block_chapauthenabled_parameter(self, chapauthenabled):