glusterd-ganesha : copy ganesha export configuration files during reboot

glusterd creates export conf file for ganesha using hook script during
volume start and ganesha_manage_export() for volume set command. But this
routine is not added in glusterd restart scenario.
Consider the following case, in a three node cluster a volume got exported
via ganesha while one of the node is offline(glusterd is not running).
When the node comes back online, that volume is not exported on that node
due to the above mentioned issue.
Also I have removed unused variables from glusterd_handle_ganesha_op()
For this patch to work pcs cluster should running on that be node.

Upstream reference
>Change-Id: I5b2312c2f3cef962b1f795b9f16c8f0a27f08ee5
>BUG: 1330097
>Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
>Reviewed-on: http://review.gluster.org/14063
>Smoke: Gluster Build System <jenkins@build.gluster.com>
>NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
>CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
>Reviewed-by: soumya k <skoduri@redhat.com>
>Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
>(cherry picked from commit f71e2fa49af185779b9f43e146effd122d4e9da0)

Change-Id: I5b2312c2f3cef962b1f795b9f16c8f0a27f08ee5
BUG: 1336801
Signed-off-by: Jiffin Tony Thottan <jthottan@redhat.com>
Reviewed-on: http://review.gluster.org/14397
Smoke: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Tested-by: Kaleb KEITHLEY <kkeithle@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>

7 files changed, 253 insertions, 69 deletions

diff --git a/extras/ganesha/scripts/Makefile.am b/extras/ganesha/scripts/Makefile.am
index 224ed26e75b..c326fc2f136 100644
--- a/extras/ganesha/scripts/Makefile.am
+++ b/extras/ganesha/scripts/Makefile.am
@@ -1,6 +1,6 @@
 EXTRA_DIST= ganesha-ha.sh dbus-send.sh create-export-ganesha.sh \
-            generate-epoch.py
+            generate-epoch.py copy-export-ganesha.sh
 
 scriptsdir = $(libexecdir)/ganesha
 scripts_SCRIPTS = create-export-ganesha.sh dbus-send.sh ganesha-ha.sh \
-                  generate-epoch.py
+                  generate-epoch.py copy-export-ganesha.sh
diff --git a/extras/ganesha/scripts/copy-export-ganesha.sh b/extras/ganesha/scripts/copy-export-ganesha.sh
new file mode 100755
index 00000000000..e8cdc98056c
--- /dev/null
+++ b/extras/ganesha/scripts/copy-export-ganesha.sh
@@ -0,0 +1,97 @@
+#/bin/bash
+
+#This script is called by glusterd when in case of
+#reboot.An export file specific to a volume
+#is copied in GANESHA_DIR/exports from online node.
+
+# Try loading the config from any of the distro
+# specific configuration locations
+if [ -f /etc/sysconfig/ganesha ]
+        then
+        . /etc/sysconfig/ganesha
+fi
+if [ -f /etc/conf.d/ganesha ]
+        then
+        . /etc/conf.d/ganesha
+fi
+if [ -f /etc/default/ganesha ]
+        then
+        . /etc/default/ganesha
+fi
+
+GANESHA_DIR=${1%/}
+VOL=$2
+CONF=
+host=$(hostname -s)
+SECRET_PEM="/var/lib/glusterd/nfs/secret.pem"
+
+function check_cmd_status()
+{
+        if [ "$1" != "0" ]
+                 then
+                 rm -rf $GANESHA_DIR/exports/export.$VOL.conf
+                 exit 1
+        fi
+}
+
+
+if [ ! -d "$GANESHA_DIR/exports" ];
+        then
+        mkdir $GANESHA_DIR/exports
+        check_cmd_status `echo $?`
+fi
+
+function find_rhel7_conf
+{
+ while [[ $# > 0 ]]
+        do
+                key="$1"
+                case $key in
+                        -f)
+                         CONFFILE="$2"
+                         ;;
+                         *)
+                         ;;
+                 esac
+                 shift
+         done
+}
+
+if [ -z $CONFFILE ]; then
+        find_rhel7_conf $OPTIONS
+
+fi
+CONF=${CONFFILE:-/etc/ganesha/ganesha.conf}
+
+#remove the old export entry from NFS-Ganesha
+#if already exported
+dbus-send --type=method_call --print-reply --system \
+          --dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr \
+          org.ganesha.nfsd.exportmgr.ShowExports \
+    | grep -w -q "/$VOL"
+if [ $? -eq 0 ]; then
+        removed_id=`cat $GANESHA_DIR/exports/export.$VOL.conf |\
+                grep Export_Id | awk -F"[=,;]" '{print$2}' | tr -d '[[:space:]]'`
+
+        dbus-send --print-reply --system --dest=org.ganesha.nfsd \
+        /org/ganesha/nfsd/ExportMgr org.ganesha.nfsd.exportmgr.RemoveExport \
+        uint16:$removed_id 2>&1
+fi
+
+ha_servers=$(pcs status | grep "Online:" | grep -o '\[.*\]' | sed -e 's/\[//' | sed -e 's/\]//')
+IFS=$' '
+for server in ${ha_servers} ; do
+        current_host=`echo $server | cut -d "." -f 1`
+        if [ $host != $current_host ]
+        then
+                scp -oPasswordAuthentication=no -oStrictHostKeyChecking=no -i \
+                ${SECRET_PEM} $server:$GANESHA_DIR/exports/export.$VOL.conf \
+                $GANESHA_DIR/exports/export.$VOL.conf
+                break
+        fi
+done
+
+if ! (cat $CONF | grep  $VOL.conf\"$ )
+then
+echo "%include \"$GANESHA_DIR/exports/export.$VOL.conf\"" >> $CONF
+fi
diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index 821cd6feb39..8d26216d908 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -456,23 +456,42 @@ create_export_config (char *volname, char **op_errstr)
                         CONFDIR, volname, NULL);
         ret = runner_run(&runner);
 
-        if (ret)
+        if (ret && op_errstr)
                 gf_asprintf (op_errstr, "Failed to create"
                             " NFS-Ganesha export config file.");
 
         return ret;
 }
 
+int
+copy_export_config (char *volname, char **op_errstr)
+{
+        runner_t                runner                     = {0,};
+        int                     ret                        = -1;
+
+        GF_ASSERT(volname);
+        runinit (&runner);
+        runner_add_args (&runner, "sh",
+                        GANESHA_PREFIX"/copy-export-ganesha.sh",
+                        CONFDIR, volname, NULL);
+        ret = runner_run(&runner);
+
+        if (ret && op_errstr)
+                gf_asprintf (op_errstr, "Failed to copy"
+                            " NFS-Ganesha export config file.");
+
+        return ret;
+}
 /* Exports and unexports a particular volume via NFS-Ganesha */
 int
-ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
+ganesha_manage_export (char *volname, char *value, char **op_errstr,
+                       gf_boolean_t reboot)
 {
         runner_t                 runner = {0,};
         int                      ret = -1;
         char                     str[1024];
         glusterd_volinfo_t      *volinfo = NULL;
         dict_t                  *vol_opts = NULL;
-        char                    *volname = NULL;
         xlator_t                *this    = NULL;
         glusterd_conf_t         *priv    = NULL;
         gf_boolean_t             option  = _gf_false;
@@ -484,16 +503,10 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
         priv = this->private;
 
         GF_ASSERT (value);
-        GF_ASSERT (dict);
         GF_ASSERT (priv);
+        GF_VALIDATE_OR_GOTO (this->name, volname, out);
+
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DICT_GET_FAILED,
-                        "Unable to get volume name");
-                goto out;
-        }
         ret = gf_string2boolean (value, &option);
         if (ret == -1) {
                 gf_msg (this->name, GF_LOG_ERROR, EINVAL,
@@ -501,54 +514,77 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
                 goto out;
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, EINVAL,
-                        GD_MSG_VOL_NOT_FOUND,
-                        FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
+        /* *
+         * Incase of reboot, following checks are already made before calling
+         * ganesha_manage_export. So it will be reductant do it again
+         */
+        if (!reboot) {
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+                                GD_MSG_VOL_NOT_FOUND,
+                                FMTSTR_CHECK_VOL_EXISTS, volname);
+                        goto out;
+                }
 
-        ret = glusterd_check_ganesha_export (volinfo);
-        if (ret && option) {
-                gf_asprintf (op_errstr, "ganesha.enable "
-                             "is already 'on'.");
-                ret = -1;
-                goto out;
+                ret = glusterd_check_ganesha_export (volinfo);
+                if (ret && option) {
+                        if (op_errstr)
+                                gf_asprintf (op_errstr, "ganesha.enable "
+                                                     "is already 'on'.");
+                        ret = -1;
+                        goto out;
 
-        }  else if (!option && !ret) {
-                gf_asprintf (op_errstr, "ganesha.enable "
-                                    "is already 'off'.");
-                ret = -1;
-                goto out;
+                }  else if (!option && !ret) {
+                        if (op_errstr)
+                                gf_asprintf (op_errstr, "ganesha.enable "
+                                                    "is already 'off'.");
+                        ret = -1;
+                        goto out;
+                }
         }
 
-        /* Check if global option is enabled, proceed only then */
-        ret = dict_get_str_boolean (priv->opts,
+        ret = 0;
+
+        /* *
+         * Incase of restart, there is chance that global option turned off
+         * with volume set command. Still we may need to clean up the
+         * configuration files.
+         * Otherwise check if global option is enabled, only then proceed
+         * */
+        if (!(reboot && !option)) {
+                ret = dict_get_str_boolean (priv->opts,
                             GLUSTERD_STORE_KEY_GANESHA_GLOBAL, _gf_false);
-        if (ret == -1) {
-                gf_msg_debug (this->name, 0, "Failed to get "
-                        "global option dict.");
-                gf_asprintf (op_errstr, "The option "
-                             "nfs-ganesha should be "
-                             "enabled before setting ganesha.enable.");
-                goto out;
-        }
-        if (!ret) {
-                gf_asprintf (op_errstr, "The option "
-                             "nfs-ganesha should be "
-                             "enabled before setting ganesha.enable.");
-                ret = -1;
-                goto out;
+                if (ret == -1) {
+                        gf_msg_debug (this->name, 0, "Failed to get "
+                                                "global option dict.");
+                        if (op_errstr)
+                                gf_asprintf (op_errstr, "The option "
+                                             "nfs-ganesha should be "
+                                             "enabled before setting "
+                                             "ganesha.enable.");
+                        goto out;
+                }
+                if (!ret) {
+                        if (op_errstr)
+                                gf_asprintf (op_errstr, "The option "
+                                             "nfs-ganesha should be "
+                                             "enabled before setting "
+                                             "ganesha.enable.");
+                        ret = -1;
+                        goto out;
+                }
         }
-
         /* Create the export file only when ganesha.enable "on" is executed */
          if (option) {
-                ret  =  create_export_config (volname, op_errstr);
+                if (reboot)
+                       ret  =  copy_export_config (volname, op_errstr);
+                else
+                       ret  =  create_export_config (volname, op_errstr);
                 if (ret) {
                         gf_msg (this->name, GF_LOG_ERROR, 0,
                                 GD_MSG_EXPORT_FILE_CREATE_FAIL,
-                                "Failed to create"
+                                "Failed to create/copy "
                                 "export file for NFS-Ganesha\n");
                         goto out;
                 }
@@ -559,21 +595,36 @@ ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
                          CONFDIR, value, volname, NULL);
                 ret = runner_run (&runner);
                 if (ret) {
-                        gf_asprintf(op_errstr, "Dynamic export"
-                                    " addition/deletion failed."
-                                    " Please see log file for details");
-                        goto out;
+                        if (op_errstr)
+                                gf_asprintf(op_errstr, "Dynamic export"
+                                            " addition/deletion failed."
+                                            " Please see log file for details");
+                        /* *
+                         * Incase of reboot scenarios, we cannot guarantee
+                         * nfs-ganesha to be running on that node, so that
+                         * dynamic export may fail
+                         */
+                        if (reboot)
+                                ret = 0;
+                        else
+                                goto out;
                 }
         }
 
-        vol_opts = volinfo->dict;
-        /* cache-invalidation should be on when a volume is exported
-         *  and off when a volume is unexported.                    */
-        ret = dict_set_dynstr_with_alloc (vol_opts,
+
+        /* *
+         * cache-invalidation should be on when a volume is exported
+         * and off when a volume is unexported. It is not required
+         * for reboot scenarios, already it will be copied.
+         * */
+        if (!reboot) {
+                vol_opts = volinfo->dict;
+                ret = dict_set_dynstr_with_alloc (vol_opts,
                                          "features.cache-invalidation", value);
-        if (ret)
-                gf_asprintf (op_errstr, "Cache-invalidation could not"
-                             " be set to %s.", value);
+                if (ret && op_errstr)
+                        gf_asprintf (op_errstr, "Cache-invalidation could not"
+                                                " be set to %s.", value);
+        }
 out:
         return ret;
 }
@@ -759,12 +810,9 @@ glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
                             char *key, char *value)
 {
 
-        int32_t                 ret          = -1;
+        int32_t                ret           = -1;
         char                   *volname      = NULL;
-        xlator_t               *this         = NULL;
         gf_boolean_t           option        = _gf_false;
-        static int             export_id     = 1;
-        glusterd_volinfo_t     *volinfo      = NULL;
 
         GF_ASSERT (dict);
         GF_ASSERT (op_errstr);
@@ -773,7 +821,15 @@ glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
 
 
         if (strcmp (key, "ganesha.enable") == 0) {
-                ret =  ganesha_manage_export (dict, value, op_errstr);
+                ret = dict_get_str (dict, "volname", &volname);
+                if (ret) {
+                        gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                                GD_MSG_DICT_GET_FAILED,
+                                "Unable to get volume name");
+                        goto out;
+                }
+                ret =  ganesha_manage_export (volname, value, op_errstr,
+                                              _gf_false);
                 if (ret < 0)
                         goto out;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index f3c0f1a5216..033fc5759f2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -2153,7 +2153,7 @@ glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
                 quorum_action = _gf_true;
         ret = glusterd_check_ganesha_export (volinfo);
         if (ret) {
-                ret = ganesha_manage_export (dict, "off", op_rspstr);
+                ret = ganesha_manage_export (volname, "off", op_rspstr, _gf_false);
                 if (ret) {
                         gf_msg (THIS->name, GF_LOG_WARNING, 0,
                                 GD_MSG_NFS_GNS_RESET_FAIL,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 5c28ec942ca..313ae89f3e6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -4060,6 +4060,9 @@ glusterd_import_friend_volume (dict_t *peer_data, size_t count)
         glusterd_volinfo_t      *old_volinfo = NULL;
         glusterd_volinfo_t      *new_volinfo = NULL;
         glusterd_svc_t          *svc         = NULL;
+        gf_boolean_t            newexportvalue;
+        gf_boolean_t            oldexportvalue;
+        char                    *value     = NULL;
 
         GF_ASSERT (peer_data);
 
@@ -4080,6 +4083,8 @@ glusterd_import_friend_volume (dict_t *peer_data, size_t count)
 
         ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
         if (0 == ret) {
+                oldexportvalue = glusterd_check_ganesha_export (old_volinfo);
+
                 /* Ref count the old_volinfo such that deleting it doesn't crash
                  * if its been already in use by other thread
                  */
@@ -4106,6 +4111,31 @@ glusterd_import_friend_volume (dict_t *peer_data, size_t count)
                 }
         }
 
+        ret = glusterd_volinfo_get (new_volinfo, "ganesha.enable", &value);
+        if (ret)
+                goto out;
+        ret = gf_string2boolean (value, &newexportvalue);
+        if (ret)
+                goto out;
+
+        /* *
+         * if new and old export value is off, then there is no point in calling
+         * ganesha_manage_export
+         */
+        if (!((newexportvalue == oldexportvalue) &&
+               newexportvalue == _gf_false)) {
+                ret = ganesha_manage_export (new_volinfo->volname, value,
+                                             NULL, _gf_true);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_NFS_GNS_OP_HANDLE_FAIL,
+                                "Returning from ganesha_manage_export with"
+                                " ret: %d for volume %s ganesha.enable %s",
+                                ret, new_volinfo->volname,
+                                value);
+                        goto out;
+                }
+        }
         ret = glusterd_store_volinfo (new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
         if (ret) {
                 gf_msg (this->name, GF_LOG_ERROR, 0,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index befe618454a..dabbdfc2e9f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1697,7 +1697,7 @@ glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
         }
         ret = glusterd_check_ganesha_export (volinfo);
         if (ret) {
-                ret = ganesha_manage_export(dict, "off", op_errstr);
+                ret = ganesha_manage_export (volname, "off", op_errstr, _gf_false);
                 if (ret) {
                         gf_msg (THIS->name, GF_LOG_WARNING, 0,
                                 GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL, "Could not "
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 65ab156e41b..3c7f27bb03a 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1067,7 +1067,8 @@ int glusterd_check_ganesha_cmd (char *key, char *value,
                                 char **errstr, dict_t *dict);
 int glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr);
 int glusterd_op_set_ganesha (dict_t *dict, char **errstr);
-int ganesha_manage_export (dict_t *dict, char *value, char **op_errstr);
+int ganesha_manage_export (char *volname, char *value, char **op_errstr,
+                           gf_boolean_t reboot);
 gf_boolean_t glusterd_check_ganesha_export (glusterd_volinfo_t *volinfo);
 int stop_ganesha (char **op_errstr);
 int tear_down_cluster (void);