summaryrefslogtreecommitdiffstats
path: root/extras
diff options
context:
space:
mode:
authorKaleb S KEITHLEY <kkeithle@redhat.com>2016-06-01 14:40:13 -0400
committerAtin Mukherjee <amukherj@redhat.com>2016-06-03 21:10:02 -0700
commit04b5886132ee0fe84011033cd2db08285cc75e31 (patch)
tree5e88c9d76bdfd8dfa9df9193fbeb1cdd75d9455c /extras
parent124425aef8118116d0bd1daa8269ab2c348b2cb9 (diff)
common-ha: race/timing issue setting up cluster
The ganesha_grace resource agent can start before the ganesha_mon resource agent, with the result that the crm_attribute that ganesha_grace expects to find has not been created yet. This is never (never? Or just so rarely that it has never actually been seen during development) seen with four nodes, but with just two nodes it's very repeatable. Note that when long (FQDN) names are used it is not unexpected to see Failed Actions in the output of `pcs status`, e.g.: * nfs-grace_monitor_5000 on node1.fully.qualified.domain.name.com 'unknown error' (1): call=20, status=complete, exitreason='none', last-rc-change='Wed Jun 1 12:32:32 2016', queued=0ms, exec=0ms * nfs-grace_monitor_5000 on node2.fully.qualified.domain.name.com 'unknown error' (1): call=18, status=complete, exitreason='none', last-rc-change='Wed Jun 1 12:32:42 2016', queued=0ms, exec=0ms and as long as all the ganesha_grace_clone and cluster_ip-1 resource agents are in Started state then this is okay. Change-Id: I726c9946ceb1ca92872b321612eb0f4c3cc039d8 BUG: 1341768 Signed-off-by: Kaleb S KEITHLEY <kkeithle@redhat.com> Reviewed-on: http://review.gluster.org/14607 Smoke: Gluster Build System <jenkins@build.gluster.com> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> CentOS-regression: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
Diffstat (limited to 'extras')
-rw-r--r--extras/ganesha/ocf/ganesha_grace25
-rw-r--r--extras/ganesha/ocf/ganesha_mon33
-rw-r--r--extras/ganesha/scripts/ganesha-ha.sh5
3 files changed, 38 insertions, 25 deletions
diff --git a/extras/ganesha/ocf/ganesha_grace b/extras/ganesha/ocf/ganesha_grace
index 21b95ddb6d7..84202df9aa2 100644
--- a/extras/ganesha/ocf/ganesha_grace
+++ b/extras/ganesha/ocf/ganesha_grace
@@ -94,15 +94,20 @@ esac
ganesha_grace_start()
{
local rc=${OCF_ERR_GENERIC}
- local short_host=$(hostname -s)
- local long_host=$(hostname)
+ local host=$(hostname -s)
ocf_log debug "ganesha_grace_start()"
- attr=$(crm_attribute --query --node=${short_host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+ # give ganesha_mon RA a chance to set the crm_attr first
+ # I mislike the sleep, but it's not clear that looping
+ # with a small sleep is necessarily better
+ # start has a 40sec timeout, so a 5sec sleep here is okay
+ sleep 5
+ attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
if [ $? -ne 0 ]; then
- attr=$(crm_attribute --query --node=${long_host} --name=${OCF_RESKEY_grace_active} 2> /dev/null )
+ host=$(hostname)
+ attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null )
if [ $? -ne 0 ]; then
- ocf_log info "crm_attribute --query --node=${short_host} --name=${OCF_RESKEY_grace_active} failed"
+ ocf_log info "grace start: crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed"
fi
fi
@@ -160,16 +165,16 @@ ganesha_grace_notify()
ganesha_grace_monitor()
{
local rc=${OCF_ERR_GENERIC}
- local short_host=$(hostname -s)
- local long_host=$(hostname)
+ local host=$(hostname -s)
ocf_log debug "monitor"
- attr=$(crm_attribute --query --node=${short_host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+ attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
if [ $? -ne 0 ]; then
- attr=$(crm_attribute --query --node=${long_host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
+ host=$(hostname)
+ attr=$(crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} 2> /dev/null)
if [ $? -ne 0 ]; then
- ocf_log info "crm_attribute --query --node=${short_host} --name=${OCF_RESKEY_grace_active} failed"
+ ocf_log info "crm_attribute --query --node=${host} --name=${OCF_RESKEY_grace_active} failed"
fi
fi
diff --git a/extras/ganesha/ocf/ganesha_mon b/extras/ganesha/ocf/ganesha_mon
index 974eb86a07a..7d0eb6b9cb8 100644
--- a/extras/ganesha/ocf/ganesha_mon
+++ b/extras/ganesha/ocf/ganesha_mon
@@ -124,8 +124,7 @@ ganesha_mon_stop()
ganesha_mon_monitor()
{
- local short_host=$(hostname -s)
- local long_host=$(hostname)
+ local host=$(hostname -s)
local pid_file="/var/run/ganesha.nfsd.pid"
# RHEL6 /etc/init.d/nfs-ganesha adds -p /var/run/ganesha.nfsd.pid
@@ -154,13 +153,15 @@ ganesha_mon_monitor()
# track grace-active crm_attr (attr != crm_attr)
# we can't just use the attr as there's no way to query
# its value in RHEL6 pacemaker
- crm_attribute --node=${short_host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
- if [ $? -ne 0 ]; then
- crm_attribute --node=${long_host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
- if [ $? -ne 0 ]; then
- ocf_log info "warning: crm_attribute --node=${short_host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed"
- fi
- fi
+
+ crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
+ if [ $? -ne 0 ]; then
+ host=$(hostname)
+ crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 2> /dev/null
+ if [ $? -ne 0 ]; then
+ ocf_log info "mon monitor warning: crm_attribute --node=${host} --lifetime=forever --name=${OCF_RESKEY_grace_active} --update=1 failed"
+ fi
+ fi
return ${OCF_SUCCESS}
fi
@@ -182,13 +183,15 @@ ganesha_mon_monitor()
ocf_log info "warning: attrd_updater -D -n ${OCF_RESKEY_grace_active} failed"
fi
- crm_attribute --node=${short_host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
+ host=$(hostname -s)
+ crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
if [ $? -ne 0 ]; then
- crm_attribute --node=${long_host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
- if [ $? -ne 0 ]; then
- ocf_log info "warning: crm_attribute --node=${short_host} --name=${OCF_RESKEY_grace_active} --update=0 failed"
- fi
- fi
+ host=$(hostname)
+ crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 2> /dev/null
+ if [ $? -ne 0 ]; then
+ ocf_log info "mon monitor warning: crm_attribute --node=${host} --name=${OCF_RESKEY_grace_active} --update=0 failed"
+ fi
+ fi
sleep ${OCF_RESKEY_grace_delay}
diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh
index f0796d2e043..f43dbf4b296 100644
--- a/extras/ganesha/scripts/ganesha-ha.sh
+++ b/extras/ganesha/scripts/ganesha-ha.sh
@@ -511,6 +511,11 @@ setup_create_resources()
logger "warning: pcs resource create nfs-mon ocf:heartbeat:ganesha_mon --clone failed"
fi
+ # see comment in (/usr/lib/ocf/resource.d/heartbeat/ganesha_grace
+ # start method. Allow time for ganesha_mon to start and set the
+ # ganesha-active crm_attribute
+ sleep 5
+
pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone meta notify=true
if [ $? -ne 0 ]; then
logger "warning: pcs resource create nfs-grace ocf:heartbeat:ganesha_grace --clone failed"