From 5392fcce7212c4cab4db23782a7834ca05c46ff6 Mon Sep 17 00:00:00 2001 From: kshithijiyer Date: Mon, 31 Aug 2020 10:20:21 +0530 Subject: [Libfix] Change logic to restart glusterd Problem: bring_bricks_online() method uses direct invocation of `service glusterd restart` at line 212. However due to change in behaviour of glusterd through patch [1] and [2] if glusterd is restart more than 6 times in an hour it goes into failed state which has to be reset using the `systemctl rest-failed` command. This causes a random failures as shown below: ``` 2020-08-25 18:00:51,600 INFO (run) root@rhs-vm24.blr.com (cp): service glusterd restart 2020-08-25 18:00:51,601 DEBUG (_get_ssh_connection) Retrieved connection from cache: root@rhs-vm24.blr.com 2020-08-25 18:00:51,830 INFO (_log_results) ^[[34;1mRETCODE (root@rhs-vm24.blr.com): 1^[[0m 2020-08-25 18:00:51,830 INFO (_log_results) ^[[31;1mSTDERR (root@rhs-vm24.blr.com)... Redirecting to /bin/systemctl restart glusterd.service Job for glusterd.service failed. See "systemctl status glusterd.service" and "journalctl -xe" for details. ^[[0m 2020-08-25 18:00:51,830 ERROR (bring_bricks_online) Unable to restart glusterd on node rhs-vm24.blr.com ``` Fix: Change the code to use restart_glusterd() from gluster_init. Links: [1] https://review.gluster.org/#/c/glusterfs/+/23751/ [2] https://review.gluster.org/#/c/glusterfs/+/23970/ Change-Id: Ibe44463ac1d444f3d2155c9ae11680c9ffd8dab9 Signed-off-by: kshithijiyer --- glustolibs-gluster/glustolibs/gluster/brick_libs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'glustolibs-gluster/glustolibs/gluster/brick_libs.py') diff --git a/glustolibs-gluster/glustolibs/gluster/brick_libs.py b/glustolibs-gluster/glustolibs/gluster/brick_libs.py index 5c056ef34..b92832dd1 100644 --- a/glustolibs-gluster/glustolibs/gluster/brick_libs.py +++ b/glustolibs-gluster/glustolibs/gluster/brick_libs.py @@ -21,6 +21,7 @@ from math import floor import time from glusto.core import Glusto as g from glustolibs.gluster.brickmux_ops import is_brick_mux_enabled +from glustolibs.gluster.gluster_init import restart_glusterd from glustolibs.gluster.volume_ops import (get_volume_info, get_volume_status) from glustolibs.gluster.volume_libs import (get_subvols, get_client_quorum_info, @@ -209,10 +210,9 @@ def bring_bricks_online(mnode, volname, bricks_list, "the bricks '%s' online", volname, bricks_list) elif bring_brick_online_method == 'glusterd_restart': - bring_brick_online_command = "service glusterd restart" brick_node, _ = brick.split(":") - ret, _, _ = g.run(brick_node, bring_brick_online_command) - if ret != 0: + ret = restart_glusterd(brick_node) + if not ret: g.log.error("Unable to restart glusterd on node %s", brick_node) _rc = False -- cgit