[Lib] Add library for reset-failed

Adding library function reset_failed_glusterd()
and modifying scratch_cleanup() to use
reset_failed_glusterd(). This is needed
because of patch [1] and [2] sent to glusterfs
where changes are made to glusterd.service.in
to not allow glusterd restart for more than 6 times
within an hour.

Links:
[1] https://review.gluster.org/#/c/glusterfs/+/23751/
[2] https://review.gluster.org/#/c/glusterfs/+/23970/

Change-Id: I25f982517420f20f11a610e8a68afc71f3b7f2a9
Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>

2 files changed, 39 insertions, 4 deletions

diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py
index d27b9ad3e..93e906fdb 100644
--- a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py
+++ b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py
@@ -45,7 +45,8 @@ from glustolibs.gluster.peer_ops import (
     peer_probe_servers, peer_status
 )
 from glustolibs.gluster.gluster_init import (
-    restart_glusterd, stop_glusterd, wait_for_glusterd_to_start)
+    restart_glusterd, stop_glusterd, wait_for_glusterd_to_start,
+    reset_failed_glusterd)
 from glustolibs.gluster.samba_libs import share_volume_over_smb
 from glustolibs.gluster.volume_libs import (
     cleanup_volume,
@@ -290,8 +291,18 @@ class GlusterBaseClass(TestCase):
                         return False
             ret = restart_glusterd(cls.servers)
             if not ret:
-                g.log.error("Failed to start glusterd")
-                return False
+                # Due to recent changes in glusterd.service.in
+                # we can only restart glusterd 6 times in an hour
+                # if not then we would need reset-failed glusterd
+                # and restart glusterd
+                ret = reset_failed_glusterd(cls.servers)
+                if not ret:
+                    g.log.error("Failed to reset glusterd")
+                    return False
+                ret = restart_glusterd(cls.servers)
+                if not ret:
+                    g.log.error("Failed to start glusterd")
+                    return False
             sleep(2)
             ret = wait_for_glusterd_to_start(cls.servers)
             if not ret:
diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_init.py b/glustolibs-gluster/glustolibs/gluster/gluster_init.py
index 29059e6a1..6e6b5aaf7 100644
--- a/glustolibs-gluster/glustolibs/gluster/gluster_init.py
+++ b/glustolibs-gluster/glustolibs/gluster/gluster_init.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-#  Copyright (C) 2015-2020  Red Hat, Inc. <http://www.redhat.com>
+#  Copyright (C) 2015-2020 Red Hat, Inc. <http://www.redhat.com>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -110,6 +110,30 @@ def restart_glusterd(servers):
     return True
 
 
+def reset_failed_glusterd(servers):
+    """Reset-failed glusterd on specified servers.
+
+    Args:
+        servers (str|list): A server|List of server hosts on which glusterd
+            has to be reset-failed.
+
+    Returns:
+        bool : True if reset-failed glusterd is successful on all servers.
+            False otherwise.
+    """
+    if not isinstance(servers, list):
+        servers = [servers]
+
+    cmd = "systemctl reset-failed glusterd"
+    results = g.run_parallel(servers, cmd)
+
+    for server, (retcode, _, _) in results.items():
+        if retcode:
+            g.log.error("Unable to reset glusterd on server %s", server)
+            return False
+    return True
+
+
 def is_glusterd_running(servers):
     """Checks the glusterd status on specified servers.