glusterd/snapshot : Copy geo-rep status and config files before taking a snapshot.

geo-rep status and conf files needs to be copied before taking a snapshot.
The idea here is, when the snapshot is restored, these config and status
files needs to be placed back in geo-replication folder so that
geo-replication can start with the same state it was when taking
a snapshot.

Details :
Before a snapshot is taken, Copy the status and config files present
in /var/lib/glusterd/geo-replication/.

The files copied are gsyncd.conf and status files of each session
belonging to a volume whose snapshot is about to be taken.

Change-Id: I0234ecd846883350c59777c2505290729de0ce05
BUG: 1061685
Signed-off-by: Sachin Pandit <spandit@redhat.com>
Reviewed-on: http://review.gluster.org/7495
Reviewed-by: Kotresh HR <khiremat@redhat.com>
Reviewed-by: Vijaikumar Mallikarjuna <vmallika@redhat.com>
Reviewed-by: Rajesh Joseph <rjoseph@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

6 files changed, 518 insertions, 2 deletions

diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index c07d1387be3..1cce6db7501 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -124,6 +124,7 @@ enum gf_common_mem_types_ {
         gf_common_mt_txn_opinfo_obj_t     = 108,
 	gf_common_mt_strfd_t              = 109,
 	gf_common_mt_strfd_data_t         = 110,
+        gf_common_mt_regex_t              = 111,
         gf_common_mt_end
 };
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index 8ef40d41a21..ad787070e8c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -26,6 +26,7 @@
 #else
 #include "mntent_compat.h"
 #endif
+#include <regex.h>
 
 #include "globals.h"
 #include "compat.h"
@@ -284,6 +285,197 @@ out:
         return ret;
 }
 
+
+/* Third argument of scandir(used in glusterd_copy_geo_rep_session_files)
+ * is filter function. As we dont want "." and ".." files present in the
+ * directory, we are excliding these 2 files.
+ * "file_select" function here does the job of filtering.
+ */
+int
+file_select (const struct dirent *entry)
+{
+        if (entry == NULL)
+                return (FALSE);
+
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+                return (FALSE);
+        else
+                return (TRUE);
+}
+
+int32_t
+glusterd_copy_geo_rep_session_files (char *session,
+                                     glusterd_volinfo_t *snap_vol)
+{
+        int32_t         ret                             = -1;
+        char            snap_session_dir[PATH_MAX]      = "";
+        char            georep_session_dir[PATH_MAX]    = "";
+        regex_t         *reg_exp                        = NULL;
+        int             file_count                      = -1;
+        struct  dirent  **files                         = {0,};
+        xlator_t        *this                           = NULL;
+        int             i                               = 0;
+        char            src_path[PATH_MAX]              = "";
+        char            dest_path[PATH_MAX]             = "";
+        glusterd_conf_t *priv                           = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (session);
+        GF_ASSERT (snap_vol);
+
+        ret = snprintf (georep_session_dir, sizeof (georep_session_dir),
+                        "%s/%s/%s", priv->workdir, GEOREP,
+                        session);
+        if (ret < 0) { /* Negative value is an error */
+                goto out;
+        }
+
+        ret = snprintf (snap_session_dir, sizeof (snap_session_dir),
+                        "%s/%s/%s/%s/%s", priv->workdir,
+                        GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                        snap_vol->snapshot->snapname, GEOREP, session);
+        if (ret < 0) { /* Negative value is an error */
+                goto out;
+        }
+
+        ret = mkdir_p (snap_session_dir, 0777, _gf_true);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Creating directory %s failed", snap_session_dir);
+                goto out;
+        }
+
+        /* TODO : good to have - Allocate in stack instead of heap */
+        reg_exp = GF_CALLOC (1, sizeof (regex_t), gf_common_mt_regex_t);
+        if (!reg_exp) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
+                        "memory for regular expression");
+                goto out;
+        }
+
+        ret = regcomp (reg_exp, "(.*status$)|(.*conf$)\0", REG_EXTENDED);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                        "compile the regular expression");
+                goto out;
+        }
+
+        /* If there are no files in a particular session then fail it*/
+        file_count = scandir (georep_session_dir, &files, file_select,
+                              alphasort);
+        if (file_count <= 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Session files not present "
+                        "in %s", georep_session_dir);
+                goto out;
+        }
+
+        /* Now compare the file name with regular expression to see if
+         * there is a match
+         */
+        for (i = 0 ; i < file_count; i++) {
+                if (regexec (reg_exp, files[i]->d_name, 0, NULL, 0))
+                        continue;
+
+                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
+                                georep_session_dir, files[i]->d_name);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = snprintf (dest_path , sizeof (dest_path), "%s/%s",
+                                snap_session_dir, files[i]->d_name);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                ret = glusterd_copy_file (src_path, dest_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not "
+                                "copy file %s of session %s",
+                                files[i]->d_name, session);
+                        goto out;
+                }
+        }
+out:
+        if (reg_exp)
+                GF_FREE (reg_exp);
+
+        return ret;
+}
+
+
+int32_t
+glusterd_copy_geo_rep_files (glusterd_volinfo_t *origin_vol,
+                             glusterd_volinfo_t *snap_vol, dict_t *rsp_dict)
+{
+        int32_t         ret                     =       -1;
+        int             i                       =       0;
+        xlator_t        *this                   =       NULL;
+        char            key[PATH_MAX]           =       "";
+        char            session[PATH_MAX]       =       "";
+        char            slave[PATH_MAX]         =       "";
+        char            snapgeo_dir[PATH_MAX]   =       "";
+        glusterd_conf_t *priv                   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (origin_vol);
+        GF_ASSERT (snap_vol);
+        GF_ASSERT (rsp_dict);
+
+        /* This condition is not satisfied if the volume
+         * is slave volume.
+         */
+        if (!origin_vol->gsync_slaves) {
+                ret = 0;
+                goto out;
+        }
+
+        GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
+
+        ret = mkdir (snapgeo_dir, 0777);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Creating directory %s failed", snapgeo_dir);
+                goto out;
+        }
+
+        for (i = 1 ; i <= origin_vol->gsync_slaves->count ; i++) {
+                ret = snprintf (key, sizeof (key), "slave%d", i);
+                if (ret < 0) /* Negative value is an error */
+                        goto out;
+
+                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
+                                                    origin_vol->gsync_slaves,
+                                                    session, slave);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get geo-rep session");
+                        goto out;
+                }
+
+                ret = glusterd_copy_geo_rep_session_files (session, snap_vol);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to copy files"
+                                " related to session %s", session);
+                        goto out;
+                }
+        }
+
+out:
+        return ret;
+}
+
 /* This function will restore a snapshot volumes
  *
  * @param dict          dictionary containing snapshot restore request
@@ -3448,6 +3640,19 @@ glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
         glusterd_auth_set_username (snap_vol, username);
         glusterd_auth_set_password (snap_vol, password);
 
+        /* TODO : Sync before taking a snapshot */
+        /* Copy the status and config files of geo-replication before
+         * taking a snapshot. During restore operation these files needs
+         * to be copied back in /var/lib/glusterd/georeplication/
+         */
+        ret = glusterd_copy_geo_rep_files (origin_vol, snap_vol, rsp_dict);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to copy geo-rep "
+                        "config and status files for volume %s",
+                        origin_vol->volname);
+                goto out;
+        }
+
         /* Adding snap brickinfos to the snap volinfo */
         brick_count = 0;
         list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
@@ -5860,6 +6065,13 @@ gd_restore_snap_volume (dict_t *rsp_dict,
                 goto out;
         }
 
+        ret = glusterd_restore_geo_rep_files (snap_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
+                        "geo-rep files");
+                goto out;
+        }
+
         /* New volinfo always shows the status as created. Therefore
          * set the status to the original volume's status. */
         glusterd_set_volume_status (new_volinfo, orig_vol->status);
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 41668b1db75..3993504e8b0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -2755,7 +2755,9 @@ glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
         glusterd_for_each_entry (entry, dir);
 
         while (entry) {
-                if ( entry->d_type != DT_DIR )
+                if ( entry->d_type != DT_DIR ||
+                   (strcmp (entry->d_name, "geo-replication") == 0
+                    && snap))
                         goto next;
 
                 volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
@@ -3299,7 +3301,6 @@ glusterd_store_retrieve_snaps (xlator_t  *this)
                                 goto out;
                         }
                 }
-
                 glusterd_for_each_entry (entry, dir);
         }
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 9c8d9940142..db08ce56495 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -568,6 +568,7 @@ glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
         new_volinfo->brick_count = volinfo->brick_count;
 
         dict_copy (volinfo->dict, new_volinfo->dict);
+        dict_copy (volinfo->gsync_slaves, new_volinfo->gsync_slaves);
         gd_update_volume_op_versions (new_volinfo);
 
         if (set_userauth) {
@@ -11607,3 +11608,287 @@ out:
         gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
         return ret;
 }
+
+int32_t
+glusterd_copy_file (const char *source, const char *destination)
+{
+        int32_t         ret             =       -1;
+        xlator_t        *this           =       NULL;
+        char            buffer[1024]    =       "";
+        int             src_fd          =       -1;
+        int             dest_fd         =       -1;
+        int             read_len        =       -1;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (source);
+        GF_ASSERT (destination);
+
+        src_fd = open (source, O_RDONLY);
+        if (src_fd < 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open file %s",
+                        source);
+                goto out;
+        }
+
+        dest_fd = open (destination, O_CREAT | O_RDWR, 755);
+        if (dest_fd < 0) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unble to open a file %s", destination);
+                goto out;
+        }
+
+        do {
+                ret = read (src_fd, buffer, sizeof (buffer));
+                if (ret ==  -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error reading file "
+                                "%s", source);
+                        goto out;
+                }
+                read_len = ret;
+                if (read_len == 0)
+                        break;
+
+                ret = write (dest_fd, buffer, read_len);
+                if (ret != read_len) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error writing in "
+                                "file %s", destination);
+                        goto out;
+                }
+        } while (ret > 0);
+out :
+        if (src_fd > 0)
+                close (src_fd);
+
+        if (dest_fd > 0)
+                close (dest_fd);
+        return ret;
+}
+
+int32_t
+glusterd_copy_folder (const char *source, const char *destination)
+{
+        DIR             *dir_ptr                =       NULL;
+        struct dirent   *direntp                =       NULL;
+        int32_t         ret                     =       -1;
+        char            src_path[PATH_MAX]      =       "";
+        char            dest_path[PATH_MAX]     =       "";
+        xlator_t        *this                   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (source);
+        GF_ASSERT (destination);
+
+        dir_ptr = opendir (source);
+        if (!dir_ptr) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s", source);
+                goto out;
+        }
+
+        while ((direntp = readdir (dir_ptr)) != NULL) {
+                if (strcmp (direntp->d_name, ".") == 0 ||
+                    strcmp (direntp->d_name, "..") == 0)
+                        continue;
+                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
+                                source, direntp->d_name);
+                if (ret < 0)
+                        goto out;
+
+                ret = snprintf (dest_path, sizeof (dest_path), "%s/%s",
+                                destination, direntp->d_name);
+                if (ret < 0)
+                        goto out;
+
+                ret = glusterd_copy_file (src_path, dest_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not copy "
+                                "%s to %s", src_path, dest_path);
+                        goto out;
+                }
+        }
+out:
+        if (dir_ptr)
+                closedir (dir_ptr);
+
+        return ret;
+}
+
+int32_t
+glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
+                              dict_t *gsync_slaves_dict, char *session,
+                              char *slave)
+{
+        int32_t         ret             =       -1;
+        char            *token          =       NULL;
+        char            *temp           =       NULL;
+        char            *ip             =       NULL;
+        char            *buffer         =       NULL;
+        xlator_t        *this           =       NULL;
+        char            *slave_temp     =       NULL;
+        char            *save_ptr       =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        GF_ASSERT (slave_key);
+        GF_ASSERT (origin_volname);
+        GF_ASSERT (gsync_slaves_dict);
+
+        ret = dict_get_str (gsync_slaves_dict, slave_key, &buffer);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                        "get value for key %s", slave_key);
+                goto out;
+        }
+
+        temp = gf_strdup (buffer);
+        if (!temp) {
+                ret = -1;
+                goto out;
+        }
+
+        token = strtok_r (temp, "/", &save_ptr);
+
+        token = strtok_r (NULL, ":", &save_ptr);
+        if (!token) {
+                ret = -1;
+                goto out;
+        }
+        token++;
+
+        ip = gf_strdup (token);
+        if (!ip) {
+                ret = -1;
+                goto out;
+        }
+
+        token = strtok_r (NULL, "\0", &save_ptr);
+        if (!token) {
+                ret = -1;
+                goto out;
+        }
+        token++;
+
+        slave_temp = gf_strdup (token);
+        if (!slave) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = snprintf (session, PATH_MAX, "%s_%s_%s",
+                        origin_volname, ip, slave_temp);
+        if (ret < 0) /* Negative value is an error */
+                goto out;
+
+        ret = snprintf  (slave, PATH_MAX, "%s::%s", ip, slave_temp);
+        if (ret < 0) {
+                goto out;
+        }
+
+        ret = 0; /* Success */
+
+out:
+        if (temp)
+                GF_FREE (temp);
+
+        if (ip)
+                GF_FREE (ip);
+
+        if (slave_temp)
+                GF_FREE (slave_temp);
+
+        return ret;
+}
+
+
+int32_t
+glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol)
+{
+        int32_t                 ret                     =       -1;
+        char                    src_path[PATH_MAX]      =       "";
+        char                    dest_path[PATH_MAX]     =       "";
+        xlator_t                *this                   =       NULL;
+        char                    *origin_volname         =       NULL;
+        glusterd_volinfo_t      *origin_vol             =       NULL;
+        int                     i                       =       0;
+        char                    key[PATH_MAX]           =       "";
+        char                    session[PATH_MAX]       =       "";
+        char                    slave[PATH_MAX]         =       "";
+        char                    snapgeo_dir[PATH_MAX]   =       "";
+        glusterd_conf_t         *priv                   =       NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_ASSERT (snap_vol);
+
+        origin_volname = gf_strdup (snap_vol->parent_volname);
+        if (!origin_volname) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (origin_volname, &origin_vol);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
+                        "volinfo for volname %s", origin_volname);
+                goto out;
+        }
+
+        for (i = 1 ; i <= snap_vol->gsync_slaves->count; i++) {
+                ret = snprintf (key, sizeof (key), "slave%d", i);
+                if (ret < 0) {
+                        goto out;
+                }
+
+                /* "origin_vol" is used here because geo-replication saves
+                 * the session in the form of master_ip_slave.
+                 * As we need the master volume to be same even after
+                 * restore, we are passing the origin volume name.
+                 *
+                 * "snap_vol->gsync_slaves" contain the slave information
+                 * when the snapshot was taken, hence we have to restore all
+                 * those slaves information when we do snapshot restore.
+                 */
+                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
+                                                    snap_vol->gsync_slaves,
+                                                    session, slave);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to get geo-rep session");
+                        goto out;
+                }
+
+                GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot,
+                                              priv);
+                ret = snprintf (src_path, sizeof (src_path),
+                                "%s/%s", snapgeo_dir, session);
+                if (ret < 0)
+                        goto out;
+
+                ret = snprintf (dest_path, sizeof (dest_path),
+                                "%s/%s/%s", priv->workdir, GEOREP,
+                                session);
+                if (ret < 0)
+                        goto out;
+
+                ret = glusterd_copy_folder (src_path, dest_path);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Could not copy "
+                                "%s to %s", src_path, dest_path);
+                        goto out;
+                }
+        }
+out:
+        if (origin_volname)
+                GF_ASSERT (origin_volname);
+
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 2d1afc379a8..9a731c4330c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -760,4 +760,17 @@ glusterd_snap_volume_remove (dict_t *rsp_dict,
 int32_t
 glusterd_store_create_snap_dir (glusterd_snap_t *snap);
 
+int32_t
+glusterd_copy_file (const char *source, const char *destination);
+
+int32_t
+glusterd_copy_folder (const char *source, const char *destination);
+
+int32_t
+glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
+                              dict_t *gsync_slaves_dict, char *session,
+                              char *slave);
+
+int32_t
+glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index cbcaecd919d..7e300d93f4d 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -495,6 +495,10 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                 snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \
                           snap->snapname);
 
+#define GLUSTERD_GET_SNAP_GEO_REP_DIR(path, snap, priv)                      \
+                snprintf (path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir,   \
+                          snap->snapname, GEOREP);
+
 #define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                           \
         if (volinfo->is_snap_volume) {                                        \
                 snprintf (path, PATH_MAX, "%s/snaps/%s/%s/%s", priv->workdir, \