glusterd/ganesha : Move ganesha-ha.conf and ganesha.conf to shared storage

Currently all the ganesha related configuration files(ganesha.conf,
ganesha-ha.conf, export files, etc) is stored locally at /etc/ganesha
on a every node in ganesha cluster. Usually we end up in two issues by
doing so :
* difficult in modifiying ganesha related conf file
* diffciult to maintain consistency of conf file across ganesha cluster
To tackle this, we plan to move all the ganesha configuration to shared
storage. As a first step in this patch ganesha.conf and ganesha-ha.conf
move to shared storage. Here actual ganesha.conf will resides in shared
stoarge and symlinks will be created in /etc/ganesha when the option
"gluster nfs-ganesha enable" is executed and remove those during the
"disable" part.

Modified prerequisites to done before running globaloption:
	* enable shared storage
	* create nfs-ganesha folder in shared storage
	* create ganesha.conf and ganesha-ha.conf in it

More details can be found at http://review.gluster.org/#/c/15105/

Change-Id: Ifabb6c5db50061f077a03932940190af74e2ca7f
BUG: 1355956
Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
Reviewed-on: http://review.gluster.org/14906
Reviewed-by: soumya k <skoduri@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

4 files changed, 126 insertions, 31 deletions

diff --git a/extras/ganesha/scripts/ganesha-ha.sh b/extras/ganesha/scripts/ganesha-ha.sh
index 143897f2401..28fc837550f 100644
--- a/extras/ganesha/scripts/ganesha-ha.sh
+++ b/extras/ganesha/scripts/ganesha-ha.sh
@@ -22,9 +22,9 @@
 
 HA_NUM_SERVERS=0
 HA_SERVERS=""
-HA_CONFDIR="/etc/ganesha"
 HA_VOL_NAME="gluster_shared_storage"
 HA_VOL_MNT="/var/run/gluster/shared_storage"
+HA_CONFDIR=$HA_VOL_MNT"/nfs-ganesha"
 SERVICE_MAN="DISTRO_NOT_FOUND"
 
 RHEL6_PCS_CNAME_OPTION="--name"
@@ -416,12 +416,8 @@ teardown_cluster()
 
 cleanup_ganesha_config ()
 {
-       rm -rf ${HA_CONFDIR}/exports/*.conf
-       rm -rf ${HA_CONFDIR}/.export_added
        rm -rf /etc/cluster/cluster.conf*
        rm -rf /var/lib/pacemaker/cib/*
-       sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' ${GANESHA_CONF}
-       rm -rf ${HA_VOL_MNT}/nfs-ganesha
 }
 
 do_create_virt_ip_constraints()
@@ -848,6 +844,29 @@ status()
     rm -f ${status_file}
 }
 
+create_ganesha_conf_file()
+{
+        if [ $1 == "yes" ];
+        then
+                if [  -e $GANESHA_CONF ];
+                then
+                        rm -rf $GANESHA_CONF
+                fi
+        # The symlink /etc/ganesha/ganesha.conf need to be
+        # created using ganesha conf file mentioned in the
+        # shared storage. Every node will only have this
+        # link and actual file will stored in shared storage,
+        # so that ganesha conf editing of ganesha conf will
+        # be easy as well as it become more consistent.
+
+                ln -s $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+        else
+        # Restoring previous file
+                rm -rf $GANESHA_CONF
+                sed -r -i -e '/^%include[[:space:]]+".+\.conf"$/d' $HA_CONFDIR/ganesha.conf
+                cp $HA_CONFDIR/ganesha.conf $GANESHA_CONF
+        fi
+}
 
 main()
 {
@@ -989,6 +1008,11 @@ $HA_CONFDIR/ganesha-ha.conf
         refresh_config ${VOL} ${HA_CONFDIR} ${HA_SERVERS}
         ;;
 
+    setup-ganesha-conf-files | --setup-ganesha-conf-files)
+
+        create_ganesha_conf_file ${1}
+        ;;
+
     *)
         # setup and teardown are not intended to be used by a
         # casual user
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index f3381e34930..23840cdb80e 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -47,7 +47,7 @@ AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(CONTRIBDIR)/userspace-rcu \
 	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
 	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \
-	-DCONFDIR=\"$(sysconfdir)/ganesha\" \
+	-DCONFDIR=\"/$(runstatedir)/gluster/shared_storage/nfs-ganesha\" \
 	-DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \
 	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index d34ec05c5f5..1becbbc7e53 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -22,8 +22,6 @@
 
 #include <ctype.h>
 
-#define SHARED_STORAGE_MNT "/var/run/gluster/shared_storage/nfs-ganesha"
-
 int start_ganesha (char **op_errstr);
 
 
@@ -267,6 +265,11 @@ glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
                                 "Could not start NFS-Ganesha");
 
                 }
+        } else {
+                ret =  stop_ganesha (op_errstr);
+                if (ret)
+                        gf_msg_debug (THIS->name, 0, "Could not stop "
+                                                "NFS-Ganesha.");
         }
 
 out:
@@ -638,8 +641,13 @@ out:
 int
 tear_down_cluster(void)
 {
-        int     ret     =       0;
-        runner_t runner =       {0,};
+        int     ret                     = 0;
+        runner_t runner                 = {0,};
+        struct stat     st              = {0,};
+        DIR            *dir             = NULL;
+        struct dirent  *entry           = NULL;
+        struct dirent   scratch[2]      = {{0,},};
+        char            path[PATH_MAX]  = {0,};
 
         if (is_ganesha_host()) {
                 runinit (&runner);
@@ -647,7 +655,55 @@ tear_down_cluster(void)
                                 GANESHA_PREFIX"/ganesha-ha.sh", "teardown",
                                 CONFDIR, NULL);
                 ret = runner_run(&runner);
+                /* *
+                 * Remove all the entries in CONFDIR expect ganesha.conf and
+                 * ganesha-ha.conf
+                 */
+                dir = sys_opendir (CONFDIR);
+                if (!dir) {
+                        gf_msg_debug (THIS->name, 0, "Failed to open directory %s. "
+                                      "Reason : %s", CONFDIR, strerror (errno));
+                        ret = 0;
+                        goto out;
+                }
+
+                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir, scratch);
+                while (entry) {
+                        snprintf (path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name);
+                        ret = sys_lstat (path, &st);
+                        if (ret == -1) {
+                                gf_msg_debug (THIS->name, 0, "Failed to stat entry %s :"
+                                              " %s", path, strerror (errno));
+                                goto out;
+                        }
+
+                        if (strcmp(entry->d_name, "ganesha.conf") == 0 ||
+                            strcmp(entry->d_name, "ganesha-ha.conf") == 0)
+                                gf_msg_debug (THIS->name, 0, " %s is not required"
+                                                " to remove", path);
+                        else if (S_ISDIR (st.st_mode))
+                                ret = recursive_rmdir (path);
+                        else
+                                ret = sys_unlink (path);
+
+                        if (ret) {
+                                gf_msg_debug (THIS->name, 0, " Failed to remove %s. "
+                                      "Reason : %s", path, strerror (errno));
+                        }
+
+                        gf_msg_debug (THIS->name, 0, "%s %s", ret ?
+                                      "Failed to remove" : "Removed", entry->d_name);
+                        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir, scratch);
+                }
+
+                ret = sys_closedir (dir);
+                if (ret) {
+                        gf_msg_debug (THIS->name, 0, "Failed to close dir %s. Reason :"
+                                      " %s", CONFDIR, strerror (errno));
+                }
         }
+
+out:
         return ret;
 }
 
@@ -685,11 +741,6 @@ teardown (char **op_errstr)
                              " HA config failed.");
                 goto out;
         }
-        ret =  stop_ganesha (op_errstr);
-        if (ret) {
-                gf_asprintf (op_errstr, "Could not stop NFS-Ganesha.");
-                goto out;
-        }
 
         runinit (&runner);
         runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
@@ -733,7 +784,17 @@ out:
 int
 stop_ganesha (char **op_errstr) {
 
-        int ret = 0;
+        int ret                 = 0;
+        runner_t runner         = {0,};
+
+        runinit (&runner);
+        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
+                         "--setup-ganesha-conf-files", CONFDIR, "no", NULL);
+        ret =  runner_run (&runner);
+        if (ret) {
+                gf_asprintf (op_errstr, "removal of symlink ganesha.conf "
+                             "in /etc/ganesha failed");
+        }
 
         if (check_host_list ()) {
                 ret = manage_service ("stop");
@@ -755,6 +816,7 @@ start_ganesha (char **op_errstr)
         int count                                          = 0;
         char *volname                                      = NULL;
         glusterd_conf_t *priv                              = NULL;
+        runner_t runner                                    = {0,};
 
         priv =  THIS->private;
         GF_ASSERT (priv);
@@ -789,6 +851,16 @@ start_ganesha (char **op_errstr)
                         goto out;
                 }
         }
+
+        runinit (&runner);
+        runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
+                         "--setup-ganesha-conf-files", CONFDIR, "yes", NULL);
+        ret =  runner_run (&runner);
+        if (ret) {
+                gf_asprintf (op_errstr, "creation of symlink ganesha.conf "
+                             "in /etc/ganesha failed");
+                goto out;
+        }
         if (check_host_list()) {
                 ret = manage_service ("start");
                 if (ret)
@@ -805,15 +877,6 @@ pre_setup (char **op_errstr)
 {
         int    ret = 0;
 
-        ret = sys_mkdir (SHARED_STORAGE_MNT, 0775);
-
-        if ((-1 == ret) && (EEXIST != errno)) {
-                gf_msg ("THIS->name", GF_LOG_ERROR, errno,
-                        GD_MSG_CREATE_DIR_FAILED, "mkdir() failed on path %s,",
-                        SHARED_STORAGE_MNT);
-                goto out;
-        }
-
         ret = check_host_list();
 
         if (ret) {
@@ -824,7 +887,6 @@ pre_setup (char **op_errstr)
                                      "Please check the log file for details");
         }
 
-out:
         return ret;
 }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index a0904fb9634..bae9be872f4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -1421,10 +1421,14 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
         char                                    *key_fixed = NULL;
         glusterd_volinfo_t                      *volinfo       = NULL;
         xlator_t                                *this = NULL;
+        glusterd_conf_t                         *priv    = NULL;
 
         this = THIS;
         GF_ASSERT (this);
 
+        priv = this->private;
+        GF_ASSERT (priv);
+
         ret = dict_get_str (dict, "volname", &volname);
 
         if (ret) {
@@ -1451,6 +1455,16 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
                 ret = glusterd_validate_volume_id (dict, volinfo);
                 if (ret)
                         goto out;
+                ret = dict_get_str_boolean (priv->opts,
+                            GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);
+                if (ret) {
+                        ret =  stop_ganesha (op_errstr);
+                        if (ret)
+                                gf_msg (THIS->name, GF_LOG_WARNING, 0,
+                                        GD_MSG_NFS_GNS_STOP_FAIL,
+                                        "Could not stop NFS-Ganesha service");
+                }
+
         }
 
         ret = dict_get_str (dict, "key", &key);
@@ -2037,11 +2051,6 @@ glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
                         gf_msg (THIS->name, GF_LOG_WARNING, errno,
                                 GD_MSG_DICT_GET_FAILED,
                                 "Could not tear down NFS-Ganesha cluster");
-                ret =  stop_ganesha (&op_errstr);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                GD_MSG_NFS_GNS_STOP_FAIL,
-                                "Could not stop NFS-Ganesha service");
         }
 
         ret = -1;