tier : Tier as a service

tierd is implemented by separating from rebalance process.

The commands affected:

1) Attach tier will trigger this process instead of old one
2) tier start and tier start force will also trigger this process.
3) volume status [tier] will show tier daemon as a process instead
of task and normal tier status and tier detach status works.
4) tier stop implemented.
5) detach tier implemented separately along with new detach tier
status
6) volume tier volname status will work using the changes.
7) volume set works

This patch has separated the tier translator from the legacy
DHT rebalance code. It now sends the RPCs from the CLI
to glusterd separate to the DHT rebalance code.
The daemon is now a service, similar to the snapshot daemon,
and can be viewed using the volume status command.

The code for the validation and commit phase are the same
as the earlier tier validation code in DHT rebalance.

The “brickop” phase has been changed so that the status
command can use this framework.

The service management framework is now used.
DHT rebalance does not use this framework.

This service framework takes care of :

*) spawning the daemon, killing it and other such processes.
*) volume set options , which are written on the volfile.
*) restart and reconfigure functions. Restart is to restart
the daemon at two points
        1)after gluster goes down and comes up.
        2) to stop detach tier.
*) reconfigure is used to make immediate volfile changes.
By doing this, we don’t restart the daemon.
it has the code to rewrite the volfile for topological
changes too (which comes into place during add and remove brick).

With this patch the log, pid, and volfile are separated
and put into respective directories.

Change-Id: I3681d0d66894714b55aa02ca2a30ac000362a399
BUG: 1313838
Signed-off-by: hari gowtham <hgowtham@redhat.com>
Reviewed-on: http://review.gluster.org/13365
Smoke: Gluster Build System <jenkins@build.gluster.org>
Tested-by: hari gowtham <hari.gowtham005@gmail.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

1 files changed, 501 insertions, 0 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c b/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c
new file mode 100644
index 00000000000..bfc879a3436
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-tierd-svc.c
@@ -0,0 +1,501 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "globals.h"
+#include "run.h"
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-tierd-svc.h"
+#include "glusterd-tierd-svc-helper.h"
+#include "glusterd-svc-helper.h"
+#include "syscall.h"
+#include "glusterd-store.h"
+
+char *tierd_svc_name = "tierd";
+
+void
+glusterd_tierdsvc_build (glusterd_svc_t *svc)
+{
+        svc->manager = glusterd_tierdsvc_manager;
+        svc->start = glusterd_tierdsvc_start;
+        svc->stop = glusterd_svc_stop;
+        svc->reconfigure = glusterd_tierdsvc_reconfigure;
+}
+
+/* a separate service framework is used because the tierd is a
+ * volume based framework while the common services are for node
+ * based daemons. when volume based common framework is available
+ * this can be consolidated into it.
+ */
+
+int
+glusterd_tierdsvc_init (void *data)
+{
+        int                     ret                = -1;
+        char                    rundir[PATH_MAX]   = {0,};
+        char                    sockpath[PATH_MAX] = {0,};
+        char                    pidfile[PATH_MAX]  = {0,};
+        char                    volfile[PATH_MAX]  = {0,};
+        char                    logdir[PATH_MAX]   = {0,};
+        char                    logfile[PATH_MAX]  = {0,};
+        char                    volfileid[256]     = {0};
+        glusterd_svc_t         *svc                = NULL;
+        glusterd_volinfo_t     *volinfo            = NULL;
+        glusterd_conf_t        *priv               = NULL;
+        glusterd_conn_notify_t  notify             = NULL;
+        xlator_t               *this               = NULL;
+        char                    *volfileserver     = NULL;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+
+        volinfo = data;
+        GF_VALIDATE_OR_GOTO (this->name, data, out);
+
+        svc = &(volinfo->tierd.svc);
+
+        ret = snprintf (svc->name, sizeof (svc->name), "%s", tierd_svc_name);
+        if (ret < 0)
+                goto out;
+
+        notify = glusterd_svc_common_rpc_notify;
+        glusterd_store_perform_node_state_store (volinfo);
+
+        glusterd_svc_build_tierd_rundir (volinfo, rundir, sizeof (rundir));
+        glusterd_svc_create_rundir (rundir);
+
+        /* Initialize the connection mgmt */
+        glusterd_svc_build_tierd_socket_filepath (volinfo, sockpath,
+                                                  sizeof (sockpath));
+        ret = glusterd_conn_init (&(svc->conn), sockpath, 600, notify);
+        if (ret)
+                goto out;
+
+        /* Initialize the process mgmt */
+        glusterd_svc_build_tierd_pidfile (volinfo, pidfile, sizeof (pidfile));
+        glusterd_svc_build_tierd_volfile_path (volinfo, volfile,
+                        sizeof (volfile));
+        glusterd_svc_build_tierd_logdir (logdir, volinfo->volname,
+                                         sizeof (logdir));
+        ret = mkdir_p (logdir, 0755, _gf_true);
+        if ((ret == -1) && (EEXIST != errno)) {
+                gf_msg (this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_CREATE_DIR_FAILED, "Unable to create logdir %s",
+                        logdir);
+                goto out;
+        }
+        glusterd_svc_build_tierd_logfile (logfile, logdir, sizeof (logfile));
+        snprintf (volfileid, sizeof (volfileid), "tierd/%s", volinfo->volname);
+
+        if (dict_get_str (this->options, "transport.socket.bind-address",
+                          &volfileserver) != 0) {
+                volfileserver = "localhost";
+        }
+        ret = glusterd_proc_init (&(svc->proc), tierd_svc_name, pidfile, logdir,
+                                  logfile, volfile, volfileid, volfileserver);
+        if (ret)
+                goto out;
+
+out:
+        gf_msg_debug (this ? this->name : "glusterd", 0, "Returning %d", ret);
+        return ret;
+}
+
+static int
+glusterd_tierdsvc_create_volfile (glusterd_volinfo_t *volinfo)
+{
+        char              filepath[PATH_MAX] = {0,};
+        int               ret                = -1;
+        glusterd_conf_t  *conf               = NULL;
+        xlator_t         *this               = NULL;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        glusterd_svc_build_tierd_volfile_path (volinfo, filepath,
+                        sizeof (filepath));
+        ret = build_rebalance_volfile (volinfo, filepath, NULL);
+
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_VOLFILE_CREATE_FAIL,
+                        "Failed to create volfile");
+                goto out;
+        }
+
+out:
+        gf_msg_debug (this ? this->name : "glusterd", 0, "Returning %d", ret);
+
+        return ret;
+
+}
+
+
+int
+glusterd_tierdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+{
+        int                 ret     = 0;
+        xlator_t           *this    = THIS;
+        glusterd_volinfo_t *volinfo = NULL;
+
+        volinfo = data;
+        GF_VALIDATE_OR_GOTO (this->name, data, out);
+
+        if (!svc->inited) {
+                ret = glusterd_tierdsvc_init (volinfo);
+                if (ret) {
+                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                                GD_MSG_TIERD_INIT_FAIL, "Failed to initialize "
+                                "tierd service for volume %s",
+                                volinfo->volname);
+                        goto out;
+                } else {
+                        svc->inited = _gf_true;
+                        gf_msg_debug (THIS->name, 0, "tierd service "
+                                      "initialized");
+                }
+        }
+
+        ret = glusterd_is_tierd_enabled (volinfo);
+        if (ret == -1) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_VOLINFO_GET_FAIL, "Failed to read volume "
+                        "options");
+                goto out;
+        }
+
+        if (ret) {
+                if (!glusterd_is_volume_started (volinfo)) {
+                        if (glusterd_proc_is_running (&svc->proc)) {
+                                ret = svc->stop (svc, SIGTERM);
+                                if (ret)
+                                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                                GD_MSG_TIERD_STOP_FAIL,
+                                                "Couldn't stop tierd for "
+                                                "volume: %s",
+                                                volinfo->volname);
+                        } else {
+                                ret = 0;
+                        }
+                        goto out;
+                }
+
+                ret = glusterd_tierdsvc_create_volfile (volinfo);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_TIERD_CREATE_FAIL, "Couldn't create "
+                                "tierd volfile for volume: %s",
+                                volinfo->volname);
+                        goto out;
+                }
+
+                ret = svc->start (svc, flags);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_TIERD_START_FAIL, "Couldn't start "
+                                "tierd for volume: %s", volinfo->volname);
+                        goto out;
+                }
+
+                glusterd_volinfo_ref (volinfo);
+                ret = glusterd_conn_connect (&(svc->conn));
+                if (ret) {
+                        glusterd_volinfo_unref (volinfo);
+                        goto out;
+                }
+
+        } else if (glusterd_proc_is_running (&svc->proc)) {
+                ret = svc->stop (svc, SIGTERM);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_TIERD_STOP_FAIL,
+                                "Couldn't stop tierd for volume: %s",
+                                volinfo->volname);
+                        goto out;
+                }
+                volinfo->tierd.port = 0;
+        }
+
+out:
+        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+
+        return ret;
+}
+
+
+int32_t
+glusterd_tierdsvc_start (glusterd_svc_t *svc, int flags)
+{
+        int                  ret                        = -1;
+        runner_t             runner                     = {0,};
+        glusterd_conf_t     *priv                       = NULL;
+        xlator_t            *this                       = NULL;
+        char                 valgrind_logfile[PATH_MAX] = {0};
+        int                  tierd_port                 = 0;
+        char                 msg[1024]                  = {0,};
+        char                 tierd_id[PATH_MAX]         = {0,};
+        glusterd_volinfo_t  *volinfo                    = NULL;
+        glusterd_tierdsvc_t *tierd                      = NULL;
+        int                  cmd                        = GF_DEFRAG_CMD_START_TIER;
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        priv = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+
+        if (glusterd_proc_is_running (&svc->proc)) {
+                ret = 0;
+                goto out;
+        }
+
+        /* Get volinfo->tierd from svc object */
+        tierd = cds_list_entry (svc, glusterd_tierdsvc_t, svc);
+        if (!tierd) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_TIERD_OBJ_GET_FAIL, "Failed to get tierd object "
+                        "from tierd service");
+                goto out;
+        }
+
+        /* Get volinfo from tierd */
+        volinfo = cds_list_entry (tierd, glusterd_volinfo_t, tierd);
+        if (!volinfo) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_VOLINFO_GET_FAIL, "Failed to get volinfo from "
+                        "from tierd");
+                goto out;
+        }
+
+        ret = sys_access (svc->proc.volfile, F_OK);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_DEBUG, 0,
+                        GD_MSG_VOLINFO_GET_FAIL,
+                        "tierd Volfile %s is not present", svc->proc.volfile);
+                /* If glusterd is down on one of the nodes and during
+                 * that time if tier is started for the first time. After some
+                 * time when the glusterd which was down comes back it tries
+                 * to look for the tierd volfile and it does not find tierd
+                 * volfile and because of this starting of tierd fails.
+                 * Therefore, if volfile is not present then create a fresh
+                 * volfile.
+                 */
+                ret = glusterd_tierdsvc_create_volfile (volinfo);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_VOLFILE_CREATE_FAIL, "Couldn't create "
+                                "tierd volfile for volume: %s",
+                                volinfo->volname);
+                        goto out;
+                }
+        }
+        runinit (&runner);
+
+        if (priv->valgrind) {
+                snprintf (valgrind_logfile, PATH_MAX, "%s/valgrind-tierd.log",
+                          svc->proc.logdir);
+
+                runner_add_args (&runner, "valgrind", "--leak-check=full",
+                                 "--trace-children=yes", "--track-origins=yes",
+                                 NULL);
+                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        }
+
+        snprintf (tierd_id, sizeof (tierd_id), "tierd-%s", volinfo->volname);
+        runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                         "-s", svc->proc.volfileserver,
+                         "--volfile-id", svc->proc.volfileid,
+                         "-p", svc->proc.pidfile,
+                         "-l", svc->proc.logfile,
+                         "--brick-name", tierd_id,
+                         "-S", svc->conn.sockpath,
+                         "--xlator-option", "*dht.use-readdirp=yes",
+                         "--xlator-option", "*dht.lookup-unhashed=yes",
+                         "--xlator-option", "*dht.assert-no-child-down=yes",
+                         "--xlator-option", "*replicate*.data-self-heal=off",
+                         "--xlator-option",
+                         "*replicate*.metadata-self-heal=off",
+                         "--xlator-option", "*replicate*.entry-self-heal=off",
+                         "--xlator-option", "*dht.readdir-optimize=on",
+                         "--xlator-option",
+                         "*tier-dht.xattr-name=trusted.tier.tier-dht",
+                         NULL);
+
+
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "*dht.rebalance-cmd=%d", cmd);
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+        runner_add_arg (&runner, "--xlator-option");
+        runner_argprintf (&runner, "*dht.commit-hash=%u",
+                          volinfo->rebal.commit_hash);
+        if (volinfo->memory_accounting)
+                runner_add_arg (&runner, "--mem-accounting");
+
+        /* Do a pmap registry remove on the older connected port */
+        if (volinfo->tierd.port) {
+                ret = pmap_registry_remove (this, volinfo->tierd.port,
+                                tierd_id, GF_PMAP_PORT_BRICKSERVER,
+                                NULL);
+                if (ret) {
+                        snprintf (msg, sizeof (msg), "Failed to remove pmap "
+                                        "registry for older signin");
+                        goto out;
+                }
+        }
+
+
+
+        tierd_port = pmap_registry_alloc (this);
+        if (!tierd_port) {
+                snprintf (msg, sizeof (msg), "Could not allocate port "
+                                "for tierd service for volume %s",
+                                volinfo->volname);
+                runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+                ret = -1;
+                goto out;
+        }
+
+        volinfo->tierd.port = tierd_port;
+
+        snprintf (msg, sizeof (msg),
+                  "Starting the tierd service for volume %s", volinfo->volname);
+        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
+
+        if (flags == PROC_START_NO_WAIT) {
+                ret = runner_run_nowait (&runner);
+        } else {
+                synclock_unlock (&priv->big_lock);
+                {
+                        ret = runner_run (&runner);
+                }
+                synclock_lock (&priv->big_lock);
+        }
+
+out:
+        return ret;
+}
+
+
+int
+glusterd_tierdsvc_restart ()
+{
+        glusterd_volinfo_t      *volinfo        = NULL;
+        int                     ret             = 0;
+        xlator_t                *this           = THIS;
+        glusterd_conf_t         *conf           = NULL;
+        glusterd_svc_t          *svc            = NULL;
+
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
+                /* Start per volume tierd svc */
+                if (volinfo->status == GLUSTERD_STATUS_STARTED &&
+                    glusterd_is_tierd_enabled (volinfo)) {
+                        svc = &(volinfo->tierd.svc);
+                        ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_TIERD_START_FAIL,
+                                        "Couldn't restart tierd for "
+                                        "vol: %s", volinfo->volname);
+                                goto out;
+                        }
+                }
+        }
+out:
+        return ret;
+}
+
+
+int
+glusterd_tierdsvc_reconfigure (void *data)
+{
+        int                     ret                     = -1;
+        xlator_t                *this                   = NULL;
+        gf_boolean_t            identical_topology      = _gf_false;
+        gf_boolean_t            identical_volfile       = _gf_false;
+        glusterd_volinfo_t      *volinfo                = NULL;
+
+        volinfo = data;
+        GF_VALIDATE_OR_GOTO (this->name, data, out);
+
+        /* reconfigure function is not available for other volume based
+         * service. but it has been implemented for tier because there can be
+         * changes on the volfile that need not be related to topology.
+         * during these changes it is better not to restart the tierd.
+         * So reconfigure is written to avoid calling restart at such
+         * situations.
+         */
+
+        this = THIS;
+        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
+
+        if (glusterd_is_tierd_enabled (volinfo))
+                goto manager;
+        /*
+         * Check both OLD and NEW volfiles, if they are SAME by size
+         * and cksum i.e. "character-by-character". If YES, then
+         * NOTHING has been changed, just return.
+         */
+
+        ret = glusterd_svc_check_tier_volfile_identical
+                (volinfo->tierd.svc.name, volinfo, &identical_volfile);
+        if (ret)
+                goto out;
+        if (identical_volfile) {
+                ret = 0;
+                goto out;
+        }
+
+        /*
+         * They are not identical. Find out if the topology is changed
+         * OR just the volume options. If just the options which got
+         * changed, then inform the xlator to reconfigure the options.
+         */
+        ret = glusterd_svc_check_tier_topology_identical
+                (volinfo->tierd.svc.name, volinfo, &identical_topology);
+        if (ret)
+                goto out; /*not able to compare due to some corruption */
+
+        /* Topology is not changed, but just the options. But write the
+         * options to tierd volfile, so that tierd will be reconfigured.
+         */
+        if (identical_topology) {
+                ret = glusterd_tierdsvc_create_volfile (volinfo);
+                if (ret == 0) {/* Only if above PASSES */
+                        ret = glusterd_fetchspec_notify (this);
+                }
+                goto out;
+        }
+        goto out;
+        /*pending add/remove brick functionality*/
+
+manager:
+        /*
+         * tierd volfile's topology has been changed. tierd server needs
+         * to be RESTARTED to ACT on the changed volfile.
+         */
+        ret = volinfo->tierd.svc.manager (&(volinfo->tierd.svc),
+                                          volinfo, PROC_START_NO_WAIT);
+
+out:
+        gf_msg_debug (THIS->name, 0, "Returning %d", ret);
+        return ret;
+}