From 1c9014d56acb709fcbaaaa5c31c9ef64069aa755 Mon Sep 17 00:00:00 2001 From: Valerii Ponomarov Date: Wed, 16 Oct 2019 18:58:09 +0530 Subject: Add possibility to check Heketi DB inconsistencies after each tc Define 'check_heketi_db_inconsistencies' config option setting there 'False' or 'True' values. Default value is 'True'. If heketi client doesn't support the 'heketi db check' feature, then 'heketi db check' just won't be performed. Change-Id: I7faff35b15e40d864c0377ae7fee154e217d8eae --- .../openshiftstoragelibs/baseclass.py | 33 ++++++++++++++++++++++ .../test_restart_gluster_services.py | 1 + tests/functional/heketi/test_heketi_metrics.py | 1 + .../heketi/test_server_state_examine_gluster.py | 1 + tests/glusterfs-containers-tests-config.yaml | 1 + 5 files changed, 37 insertions(+) diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py index 1e52f560..b000a643 100644 --- a/openshift-storage-libs/openshiftstoragelibs/baseclass.py +++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py @@ -17,6 +17,7 @@ from openshiftstoragelibs.heketi_ops import ( hello_heketi, heketi_blockvolume_delete, heketi_blockvolume_info, + heketi_db_check, heketi_volume_create, heketi_volume_delete, heketi_volume_info, @@ -60,6 +61,9 @@ class BaseClass(unittest.TestCase): ERROR_OR_FAILURE_EXISTS = False STOP_ON_FIRST_FAILURE = bool(g.config.get("common", {}).get( "stop_on_first_failure", False)) + CHECK_HEKETI_DB_INCONSISTENCIES = ( + g.config.get("common", {}).get("check_heketi_db_inconsistencies", True) + in (True, 'TRUE', 'True', 'true', 'yes', 'Yes', 'YES')) @classmethod def setUpClass(cls): @@ -130,6 +134,17 @@ class BaseClass(unittest.TestCase): "to one test case failure.") super(BaseClass, self).setUp() + if self.CHECK_HEKETI_DB_INCONSISTENCIES: + try: + self.heketi_db_inconsistencies = heketi_db_check( + self.heketi_client_node, self.heketi_server_url) + except NotImplementedError as e: + g.log.info("Can not check Heketi DB inconsistencies due to " + "the following error: %s" % e) + else: + self.addCleanup( + self.check_heketi_db_inconsistencies, + self.heketi_db_inconsistencies["totalinconsistencies"]) msg = "Starting Test : %s : %s" % (self.id(), self.glustotest_run_id) g.log.info(msg) @@ -151,6 +166,24 @@ class BaseClass(unittest.TestCase): return command.cmd_run( cmd=cmd, hostname=hostname, raise_on_error=raise_on_error) + def check_heketi_db_inconsistencies( + self, number_of_allowed_heketi_db_inconsistencies): + current_heketi_db_inconsistencies = heketi_db_check( + self.heketi_client_node, self.heketi_server_url) + current_number_of_heketi_db_inconsistencies = ( + current_heketi_db_inconsistencies["totalinconsistencies"]) + error_msg = ( + "Before the test case we had %s inconsistencies, but after " + "the test case we have %s inconsistencies in the Heketi DB.\n" + "'heketi-cli db check' command output is following:\n%s" % ( + number_of_allowed_heketi_db_inconsistencies, + current_number_of_heketi_db_inconsistencies, + current_heketi_db_inconsistencies)) + self.assertEqual( + number_of_allowed_heketi_db_inconsistencies, + current_number_of_heketi_db_inconsistencies, + error_msg) + def create_secret(self, secret_name_prefix="autotests-secret"): secret_name = oc_create_secret( self.ocp_client[0], diff --git a/tests/functional/gluster_stability/test_restart_gluster_services.py b/tests/functional/gluster_stability/test_restart_gluster_services.py index 3d760f27..aaa16d51 100644 --- a/tests/functional/gluster_stability/test_restart_gluster_services.py +++ b/tests/functional/gluster_stability/test_restart_gluster_services.py @@ -51,6 +51,7 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass): """Deploys, Verifies and adds resources required for testcases in cleanup method """ + super(GlusterStabilityTestSetup, self).setUp() self.oc_node = self.ocp_master_node[0] self.prefix = "autotest-%s" % utils.get_random_str() diff --git a/tests/functional/heketi/test_heketi_metrics.py b/tests/functional/heketi/test_heketi_metrics.py index 9e4f5ff8..65a8c2ec 100644 --- a/tests/functional/heketi/test_heketi_metrics.py +++ b/tests/functional/heketi/test_heketi_metrics.py @@ -19,6 +19,7 @@ from openshiftstoragelibs.openshift_ops import ( class TestHeketiMetrics(BaseClass): def setUp(self): + super(TestHeketiMetrics, self).setUp() self.node = self.ocp_master_node[0] version = heketi_version.get_heketi_version(self.heketi_client_node) if version < '6.0.0-14': diff --git a/tests/functional/heketi/test_server_state_examine_gluster.py b/tests/functional/heketi/test_server_state_examine_gluster.py index 31859bd0..427fda89 100644 --- a/tests/functional/heketi/test_server_state_examine_gluster.py +++ b/tests/functional/heketi/test_server_state_examine_gluster.py @@ -7,6 +7,7 @@ from openshiftstoragelibs import openshift_ops class TestHeketiServerStateExamineGluster(BaseClass): def setUp(self): + super(TestHeketiServerStateExamineGluster, self).setUp() self.node = self.ocp_master_node[0] version = heketi_version.get_heketi_version(self.heketi_client_node) if version < '8.0.0-7': diff --git a/tests/glusterfs-containers-tests-config.yaml b/tests/glusterfs-containers-tests-config.yaml index c99e9c14..419aaeca 100644 --- a/tests/glusterfs-containers-tests-config.yaml +++ b/tests/glusterfs-containers-tests-config.yaml @@ -76,6 +76,7 @@ openshift: common: allow_heketi_zones_update: False + check_heketi_db_inconsistencies: True stop_on_first_failure: False heketi_command_timeout: 120 -- cgit