From 7a33280b5ef65edb8845a19332938c69e1fea003 Mon Sep 17 00:00:00 2001 From: kshithijiyer Date: Wed, 17 Jun 2020 13:37:38 +0530 Subject: [Libfix] Add retry logic to restart_glusterd() Problem: Patch [1] and [2] sent to glusterfs where changes are made to glusterd.service.in to not allow glusterd restart for more than 6 times within an hour, due this glusterd restarts present in testcases may fail as there is no way to figure out when we reach the 6 restart limit. Fix: Add code to check if glusterd restart has failed if true then call reset_failed_glusterd(), and redo the restart. Links: [1] https://review.gluster.org/#/c/glusterfs/+/23751/ [2] https://review.gluster.org/#/c/glusterfs/+/23970/ Change-Id: I041a019f9a8757d8fead00302e6bbcd6563dc74e Signed-off-by: kshithijiyer --- .../glustolibs/gluster/gluster_base_class.py | 17 +++-------------- glustolibs-gluster/glustolibs/gluster/gluster_init.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py index 93e906fdb..d27b9ad3e 100644 --- a/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py +++ b/glustolibs-gluster/glustolibs/gluster/gluster_base_class.py @@ -45,8 +45,7 @@ from glustolibs.gluster.peer_ops import ( peer_probe_servers, peer_status ) from glustolibs.gluster.gluster_init import ( - restart_glusterd, stop_glusterd, wait_for_glusterd_to_start, - reset_failed_glusterd) + restart_glusterd, stop_glusterd, wait_for_glusterd_to_start) from glustolibs.gluster.samba_libs import share_volume_over_smb from glustolibs.gluster.volume_libs import ( cleanup_volume, @@ -291,18 +290,8 @@ class GlusterBaseClass(TestCase): return False ret = restart_glusterd(cls.servers) if not ret: - # Due to recent changes in glusterd.service.in - # we can only restart glusterd 6 times in an hour - # if not then we would need reset-failed glusterd - # and restart glusterd - ret = reset_failed_glusterd(cls.servers) - if not ret: - g.log.error("Failed to reset glusterd") - return False - ret = restart_glusterd(cls.servers) - if not ret: - g.log.error("Failed to start glusterd") - return False + g.log.error("Failed to start glusterd") + return False sleep(2) ret = wait_for_glusterd_to_start(cls.servers) if not ret: diff --git a/glustolibs-gluster/glustolibs/gluster/gluster_init.py b/glustolibs-gluster/glustolibs/gluster/gluster_init.py index 6e6b5aaf7..fd8f174ed 100644 --- a/glustolibs-gluster/glustolibs/gluster/gluster_init.py +++ b/glustolibs-gluster/glustolibs/gluster/gluster_init.py @@ -81,13 +81,17 @@ def stop_glusterd(servers): return True -def restart_glusterd(servers): +def restart_glusterd(servers, enable_retry=True): """Restart the glusterd on specified servers. Args: servers (str|list): A server|List of server hosts on which glusterd has to be restarted. + Kwargs: + enable_retry(Bool): If set to True than runs reset-failed else + do nothing. + Returns: bool : True if restarting glusterd is successful on all servers. False otherwise. @@ -104,10 +108,12 @@ def restart_glusterd(servers): if retcode != 0: g.log.error("Unable to restart glusterd on server %s", server) _rc = False - if not _rc: - return False - - return True + if not _rc and enable_retry: + ret = reset_failed_glusterd(servers) + if ret: + ret = restart_glusterd(servers) + return ret + return _rc def reset_failed_glusterd(servers): -- cgit