[Libfix] Add retry logic to restart_glusterd()

Problem:
Patch [1] and [2] sent to glusterfs
where changes are made to glusterd.service.in
to not allow glusterd restart for more than 6 times
within an hour, due this glusterd restarts
present in testcases may fail as there is no way
to figure out when we reach the 6 restart limit.

Fix:
Add code to check if glusterd restart has failed
if true then call reset_failed_glusterd(), and redo
the restart.

Links:
[1] https://review.gluster.org/#/c/glusterfs/+/23751/
[2] https://review.gluster.org/#/c/glusterfs/+/23970/

Change-Id: I041a019f9a8757d8fead00302e6bbcd6563dc74e
Signed-off-by: kshithijiyer <kshithij.ki@gmail.com>

1 files changed, 11 insertions, 5 deletions

diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_init.py b/glustolibs-gluster/glustolibs/gluster/gluster_init.py
index 6e6b5aaf7..fd8f174ed 100644
--- a/glustolibs-gluster/glustolibs/gluster/gluster_init.py
+++ b/glustolibs-gluster/glustolibs/gluster/gluster_init.py
@@ -81,13 +81,17 @@ def stop_glusterd(servers):
     return True
 
 
-def restart_glusterd(servers):
+def restart_glusterd(servers, enable_retry=True):
     """Restart the glusterd on specified servers.
 
     Args:
         servers (str|list): A server|List of server hosts on which glusterd
             has to be restarted.
 
+    Kwargs:
+        enable_retry(Bool): If set to True than runs reset-failed else
+                            do nothing.
+
     Returns:
         bool : True if restarting glusterd is successful on all servers.
             False otherwise.
@@ -104,10 +108,12 @@ def restart_glusterd(servers):
         if retcode != 0:
             g.log.error("Unable to restart glusterd on server %s", server)
             _rc = False
-    if not _rc:
-        return False
-
-    return True
+    if not _rc and enable_retry:
+        ret = reset_failed_glusterd(servers)
+        if ret:
+            ret = restart_glusterd(servers)
+        return ret
+    return _rc
 
 
 def reset_failed_glusterd(servers):