Add possibility to check Heketi DB inconsistencies after each tc

Define 'check_heketi_db_inconsistencies' config option setting
there 'False' or 'True' values. Default value is 'True'.
If heketi client doesn't support the 'heketi db check' feature, then
'heketi db check' just won't be performed.

Change-Id: I7faff35b15e40d864c0377ae7fee154e217d8eae

5 files changed, 37 insertions, 0 deletions

diff --git a/openshift-storage-libs/openshiftstoragelibs/baseclass.py b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
index 1e52f560..b000a643 100644
--- a/openshift-storage-libs/openshiftstoragelibs/baseclass.py
+++ b/openshift-storage-libs/openshiftstoragelibs/baseclass.py
@@ -17,6 +17,7 @@ from openshiftstoragelibs.heketi_ops import (
     hello_heketi,
     heketi_blockvolume_delete,
     heketi_blockvolume_info,
+    heketi_db_check,
     heketi_volume_create,
     heketi_volume_delete,
     heketi_volume_info,
@@ -60,6 +61,9 @@ class BaseClass(unittest.TestCase):
     ERROR_OR_FAILURE_EXISTS = False
     STOP_ON_FIRST_FAILURE = bool(g.config.get("common", {}).get(
         "stop_on_first_failure", False))
+    CHECK_HEKETI_DB_INCONSISTENCIES = (
+        g.config.get("common", {}).get("check_heketi_db_inconsistencies", True)
+        in (True, 'TRUE', 'True', 'true', 'yes', 'Yes', 'YES'))
 
     @classmethod
     def setUpClass(cls):
@@ -130,6 +134,17 @@ class BaseClass(unittest.TestCase):
                           "to one test case failure.")
 
         super(BaseClass, self).setUp()
+        if self.CHECK_HEKETI_DB_INCONSISTENCIES:
+            try:
+                self.heketi_db_inconsistencies = heketi_db_check(
+                    self.heketi_client_node, self.heketi_server_url)
+            except NotImplementedError as e:
+                g.log.info("Can not check Heketi DB inconsistencies due to "
+                           "the following error: %s" % e)
+            else:
+                self.addCleanup(
+                    self.check_heketi_db_inconsistencies,
+                    self.heketi_db_inconsistencies["totalinconsistencies"])
 
         msg = "Starting Test : %s : %s" % (self.id(), self.glustotest_run_id)
         g.log.info(msg)
@@ -151,6 +166,24 @@ class BaseClass(unittest.TestCase):
         return command.cmd_run(
             cmd=cmd, hostname=hostname, raise_on_error=raise_on_error)
 
+    def check_heketi_db_inconsistencies(
+            self, number_of_allowed_heketi_db_inconsistencies):
+        current_heketi_db_inconsistencies = heketi_db_check(
+            self.heketi_client_node, self.heketi_server_url)
+        current_number_of_heketi_db_inconsistencies = (
+            current_heketi_db_inconsistencies["totalinconsistencies"])
+        error_msg = (
+            "Before the test case we had %s inconsistencies, but after "
+            "the test case we have %s inconsistencies in the Heketi DB.\n"
+            "'heketi-cli db check' command output is following:\n%s" % (
+                number_of_allowed_heketi_db_inconsistencies,
+                current_number_of_heketi_db_inconsistencies,
+                current_heketi_db_inconsistencies))
+        self.assertEqual(
+            number_of_allowed_heketi_db_inconsistencies,
+            current_number_of_heketi_db_inconsistencies,
+            error_msg)
+
     def create_secret(self, secret_name_prefix="autotests-secret"):
         secret_name = oc_create_secret(
             self.ocp_client[0],
diff --git a/tests/functional/gluster_stability/test_restart_gluster_services.py b/tests/functional/gluster_stability/test_restart_gluster_services.py
index 3d760f27..aaa16d51 100644
--- a/tests/functional/gluster_stability/test_restart_gluster_services.py
+++ b/tests/functional/gluster_stability/test_restart_gluster_services.py
@@ -51,6 +51,7 @@ class GlusterStabilityTestSetup(GlusterBlockBaseClass):
         """Deploys, Verifies and adds resources required for testcases
            in cleanup method
         """
+        super(GlusterStabilityTestSetup, self).setUp()
         self.oc_node = self.ocp_master_node[0]
         self.prefix = "autotest-%s" % utils.get_random_str()
 
diff --git a/tests/functional/heketi/test_heketi_metrics.py b/tests/functional/heketi/test_heketi_metrics.py
index 9e4f5ff8..65a8c2ec 100644
--- a/tests/functional/heketi/test_heketi_metrics.py
+++ b/tests/functional/heketi/test_heketi_metrics.py
@@ -19,6 +19,7 @@ from openshiftstoragelibs.openshift_ops import (
 class TestHeketiMetrics(BaseClass):
 
     def setUp(self):
+        super(TestHeketiMetrics, self).setUp()
         self.node = self.ocp_master_node[0]
         version = heketi_version.get_heketi_version(self.heketi_client_node)
         if version < '6.0.0-14':
diff --git a/tests/functional/heketi/test_server_state_examine_gluster.py b/tests/functional/heketi/test_server_state_examine_gluster.py
index 31859bd0..427fda89 100644
--- a/tests/functional/heketi/test_server_state_examine_gluster.py
+++ b/tests/functional/heketi/test_server_state_examine_gluster.py
@@ -7,6 +7,7 @@ from openshiftstoragelibs import openshift_ops
 class TestHeketiServerStateExamineGluster(BaseClass):
 
     def setUp(self):
+        super(TestHeketiServerStateExamineGluster, self).setUp()
         self.node = self.ocp_master_node[0]
         version = heketi_version.get_heketi_version(self.heketi_client_node)
         if version < '8.0.0-7':
diff --git a/tests/glusterfs-containers-tests-config.yaml b/tests/glusterfs-containers-tests-config.yaml
index c99e9c14..419aaeca 100644
--- a/tests/glusterfs-containers-tests-config.yaml
+++ b/tests/glusterfs-containers-tests-config.yaml
@@ -76,6 +76,7 @@ openshift:
 
 common:
     allow_heketi_zones_update: False
+    check_heketi_db_inconsistencies: True
     stop_on_first_failure: False
     heketi_command_timeout: 120