Add possibility to skip test cases and cleanups after failure

In some cases, it is useful to stop test execution after first failure.
Not only stop it, but also skip all the scheduled cleanups.
It will allow us to keep a cluster in the best state for a root cause
debugging.

Just define 'common.stop_on_first_failure' option to True value
in the config file. It will make the test runner stop after first
failure skipping all the scheduled cleanups and pending test cases.

Change-Id: I963eb038a11a8e2088a84f7ba4838870ea3e657a

3 files changed, 69 insertions, 4 deletions

diff --git a/cns-libs/cnslibs/cns/cns_baseclass.py b/cns-libs/cnslibs/cns/cns_baseclass.py
index 4df46901..9e7912b2 100644
--- a/cns-libs/cnslibs/cns/cns_baseclass.py
+++ b/cns-libs/cnslibs/cns/cns_baseclass.py
@@ -1,8 +1,8 @@
 import datetime
-import unittest
 
 from glusto.core import Glusto as g
 
+from cnslibs.common import baseclass
 from cnslibs.common import command
 from cnslibs.common.exceptions import ExecutionError
 from cnslibs.common.heketi_ops import (
@@ -24,7 +24,7 @@ from cnslibs.common.openshift_ops import (
 )
 
 
-class BaseClass(unittest.TestCase):
+class BaseClass(baseclass.BaseClass):
     """Base class for test classes."""
 
     @classmethod
diff --git a/cns-libs/cnslibs/common/baseclass.py b/cns-libs/cnslibs/common/baseclass.py
new file mode 100644
index 00000000..36f00ff6
--- /dev/null
+++ b/cns-libs/cnslibs/common/baseclass.py
@@ -0,0 +1,65 @@
+import unittest
+
+from glusto.core import Glusto as g
+
+
+class BaseClass(unittest.TestCase):
+
+    ERROR_OR_FAILURE_EXISTS = False
+    STOP_ON_FIRST_FAILURE = bool(g.config.get("common", {}).get(
+        "stop_on_first_failure", False))
+
+    def setUp(self):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and
+                BaseClass.ERROR_OR_FAILURE_EXISTS):
+            self.skipTest("Test is skipped, because of the restriction "
+                          "to one test case failure.")
+        return super(BaseClass, self).setUp()
+
+    def _is_error_or_failure_exists(self):
+        if hasattr(self, '_outcome'):
+            # Python 3.4+
+            result = self.defaultTestResult()
+            self._feedErrorsToResult(result, self._outcome.errors)
+        else:
+            # Python 2.7-3.3
+            result = getattr(
+                self, '_outcomeForDoCleanups', self._resultForDoCleanups)
+        ok_result = True
+        for attr in ('errors', 'failures'):
+            if not hasattr(result, attr):
+                continue
+            exc_list = getattr(result, attr)
+            if exc_list and exc_list[-1][0] is self:
+                ok_result = ok_result and not exc_list[-1][1]
+        if hasattr(result, '_excinfo'):
+            ok_result = ok_result and not result._excinfo
+        if ok_result:
+            return False
+        self.ERROR_OR_FAILURE_EXISTS = True
+        BaseClass.ERROR_OR_FAILURE_EXISTS = True
+        return True
+
+    def doCleanups(self):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and (
+                self.ERROR_OR_FAILURE_EXISTS or
+                self._is_error_or_failure_exists())):
+            while self._cleanups:
+                (func, args, kwargs) = self._cleanups.pop()
+                msg = ("Found test case failure. Avoiding run of scheduled "
+                       "following cleanup:\nfunc = %s\nargs = %s\n"
+                       "kwargs = %s" % (func, args, kwargs))
+                g.log.warn(msg)
+        return super(BaseClass, self).doCleanups()
+
+    @classmethod
+    def doClassCleanups(cls):
+        if (BaseClass.STOP_ON_FIRST_FAILURE and
+                BaseClass.ERROR_OR_FAILURE_EXISTS):
+            while cls._class_cleanups:
+                (func, args, kwargs) = cls._class_cleanups.pop()
+                msg = ("Found test case failure. Avoiding run of scheduled "
+                       "following cleanup:\nfunc = %s\nargs = %s\n"
+                       "kwargs = %s" % (func, args, kwargs))
+                g.log.warn(msg)
+        return super(BaseClass, cls).doClassCleanups()
diff --git a/cns-libs/cnslibs/common/heketi_libs.py b/cns-libs/cnslibs/common/heketi_libs.py
index 745ab229..1c86776c 100644
--- a/cns-libs/cnslibs/common/heketi_libs.py
+++ b/cns-libs/cnslibs/common/heketi_libs.py
@@ -1,8 +1,8 @@
 import datetime
-import unittest
 
 from glusto.core import Glusto as g
 
+from cnslibs.common import baseclass
 from cnslibs.common.exceptions import ExecutionError, ConfigError
 from cnslibs.common.heketi_ops import (hello_heketi,
                                        heketi_volume_delete,
@@ -10,7 +10,7 @@ from cnslibs.common.heketi_ops import (hello_heketi,
 from cnslibs.common import openshift_ops
 
 
-class HeketiBaseClass(unittest.TestCase):
+class HeketiBaseClass(baseclass.BaseClass):
     """
     This class initializes heketi config variables, constructs topology info
     dictionary and check if heketi server is alive.