mgmt/glusterd: Consider nodesvc to be running after online

Definition of online in the message below is that the
RPC_CLNT_CONNECT event arrives for the nfs/self-heal-daemon process.

For automated tests, sometimes the script needs to wait until
self-heal-daemon comes online, so that the relevant
commands can be executed. Gluster volume status before this change
printed whether the self-heal-daemon is running or not based on the
lock availability on the pidfile. But there is a small window where
the lock on pid file is present but the process is still not
online. So the commands that were depending on this kept failing in
the test script.

Change-Id: I0e44e18b08d7b653d34fa170c1f187d91c888cd9
BUG: 858212
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
Reviewed-on: http://review.gluster.org/4236
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

7 files changed, 23 insertions, 21 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 1b1d113b5..673bfaeb7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -2997,14 +2997,14 @@ glusterd_nodesvc_rpc_notify (struct rpc_clnt *rpc, void *mydata,
         switch (event) {
         case RPC_CLNT_CONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-                (void) glusterd_nodesvc_set_running (server, _gf_true);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_true);
                 ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
 
         case RPC_CLNT_DISCONNECT:
                 gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_DISCONNECT");
-                (void) glusterd_nodesvc_set_running (server, _gf_false);
+                (void) glusterd_nodesvc_set_online_status (server, _gf_false);
                 break;
 
         default:
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index ac53f2e46..48ae5b660 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -4134,7 +4134,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATS_INFO:
                 ret = dict_get_str_boolean (dict, "nfs", _gf_false);
                 if (ret) {
-                        if (!glusterd_nodesvc_is_running ("nfs")) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
                                 ret = -1;
                                 gf_log (this->name, GF_LOG_ERROR, "NFS server"
                                         " is not running");
@@ -4177,7 +4177,7 @@ glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr)
         case GF_CLI_STATS_TOP:
                 ret = dict_get_str_boolean (dict, "nfs", _gf_false);
                 if (ret) {
-                        if (!glusterd_nodesvc_is_running ("nfs")) {
+                        if (!glusterd_is_nodesvc_online ("nfs")) {
                                 ret = -1;
                                 gf_log (this->name, GF_LOG_ERROR, "NFS server"
                                         " is not running");
@@ -4614,7 +4614,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
                 ret = 0;
         } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                if (!glusterd_nodesvc_is_running ("nfs")) {
+                if (!glusterd_is_nodesvc_online ("nfs")) {
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "NFS server is not running");
@@ -4633,7 +4633,7 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr)
 
                 ret = 0;
         } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (!glusterd_nodesvc_is_running ("glustershd")) {
+                if (!glusterd_is_nodesvc_online ("glustershd")) {
                         ret = -1;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Self-heal daemon is not running");
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 5395e0a9c..879bb126a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -3090,7 +3090,7 @@ glusterd_get_nodesvc_volfile (char *server, char *workdir,
 }
 
 void
-glusterd_nodesvc_set_running (char *server, gf_boolean_t status)
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status)
 {
         glusterd_conf_t *priv = NULL;
 
@@ -3101,16 +3101,16 @@ glusterd_nodesvc_set_running (char *server, gf_boolean_t status)
         GF_ASSERT (priv->nfs);
 
         if (!strcmp("glustershd", server))
-                priv->shd->running = status;
+                priv->shd->online = status;
         else if (!strcmp ("nfs", server))
-                priv->nfs->running = status;
+                priv->nfs->online = status;
 }
 
 gf_boolean_t
-glusterd_nodesvc_is_running (char *server)
+glusterd_is_nodesvc_online (char *server)
 {
         glusterd_conf_t *conf = NULL;
-        gf_boolean_t    running = _gf_false;
+        gf_boolean_t    online = _gf_false;
 
         GF_ASSERT (server);
         conf = THIS->private;
@@ -3119,11 +3119,11 @@ glusterd_nodesvc_is_running (char *server)
         GF_ASSERT (conf->nfs);
 
         if (!strcmp (server, "glustershd"))
-                running = conf->shd->running;
+                online = conf->shd->online;
         else if (!strcmp (server, "nfs"))
-                running = conf->nfs->running;
+                online = conf->nfs->online;
 
-        return running;
+        return online;
 }
 
 int32_t
@@ -3418,7 +3418,7 @@ glusterd_nodesvc_stop (char *server, int sig)
         ret = glusterd_service_stop (server, pidfile, sig, _gf_true);
 
         if (ret == 0) {
-                glusterd_nodesvc_set_running (server, _gf_false);
+                glusterd_nodesvc_set_online_status (server, _gf_false);
                 (void)glusterd_nodesvc_unlink_socket_file (server);
         }
 out:
@@ -3492,7 +3492,9 @@ glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
 
         glusterd_get_nodesvc_pidfile (server, priv->workdir, pidfile,
                                       sizeof (pidfile));
-        running = glusterd_is_service_running (pidfile, &pid);
+        //Consider service to be running only when glusterd sees it Online
+        if (glusterd_is_nodesvc_online (server))
+                running = glusterd_is_service_running (pidfile, &pid);
 
         /* For nfs-servers/self-heal-daemon setting
          * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 00353b03e..c90c9d918 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -208,10 +208,10 @@ int32_t
 glusterd_nodesvc_connect (char *server, char *socketpath);
 
 void
-glusterd_nodesvc_set_running (char *server, gf_boolean_t status);
+glusterd_nodesvc_set_online_status (char *server, gf_boolean_t status);
 
 gf_boolean_t
-glusterd_nodesvc_is_running (char *server);
+glusterd_is_nodesvc_online (char *server);
 
 int
 glusterd_remote_hostname_get (rpcsvc_request_t *req,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index db143d5f0..263c8bf01 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1102,7 +1102,7 @@ glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        if (!glusterd_nodesvc_is_running ("glustershd")) {
+        if (!glusterd_is_nodesvc_online ("glustershd")) {
                 ret = -1;
                 snprintf (msg, sizeof (msg), "Self-heal daemon is not "
                           "running. Check self-heal daemon log file.");
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index f9cdb65ae..204a6adfc 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -1057,7 +1057,7 @@ init (xlator_t *this)
 #endif
 
         this->private = conf;
-        (void) glusterd_nodesvc_set_running ("glustershd", _gf_false);
+        (void) glusterd_nodesvc_set_online_status ("glustershd", _gf_false);
 
         GLUSTERD_GET_HOOKS_DIR (hooks_dir, GLUSTERD_HOOK_VER, conf);
         if (stat (hooks_dir, &buf)) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 36cd8f8fc..66adff741 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -96,7 +96,7 @@ struct glusterd_volgen {
 
 typedef struct {
         struct rpc_clnt         *rpc;
-        gf_boolean_t            running;
+        gf_boolean_t            online;
 } nodesrv_t;
 
 #define GD_OP_VERSION_KEY     "operating-version"