cluster/tier: add pause tier for snapshots

This is a backport of 12304

Snaps of tiered volumes cannot handle files undergoing migration.
We implement a helper mechanism to "pause" migration. Any files
undergoing migration are aborted. Clean up is done to remove
sticky bits and data at the destination. Migration is restarted
after snap completes.

For testing an internal switch is added. It is not exposed externally.

gluster volume set vol1 tier-pause [true|false]

> Change-Id: Ia85bbf89ac142e9b7e73fcbef98bb9da86097799
> BUG: 1267950
> Signed-off-by: Dan Lambright <dlambrig@redhat.com>
> Reviewed-on: http://review.gluster.org/12304
> Reviewed-by: N Balachandran <nbalacha@redhat.com>
> Tested-by: NetBSD Build System <jenkins@build.gluster.org>
> Tested-by: Gluster Build System <jenkins@build.gluster.com>
Signed-off-by: Dan Lambright <dlambrig@redhat.com>

Conflicts:
	xlators/mgmt/glusterd/src/glusterd-messages.h

Change-Id: I5f039d8d38a4c915bd873969f336b96755a0b8f1
BUG: 1274101
Reviewed-on: http://review.gluster.org/12411
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Tested-by: Dan Lambright <dlambrig@redhat.com>

3 files changed, 208 insertions, 3 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index d67837240e4..9b3f5178bde 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -45,7 +45,9 @@
  */
 
 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
+
 #define GLFS_NUM_MESSAGES       568
+
 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
 #define glfs_msg_start_x GLFS_COMP_BASE, "Invalid: Start of messages"
@@ -4610,6 +4612,32 @@
  */
 #define GD_MSG_SVC_STOP_FAIL                        (GLUSTERD_COMP_BASE + 570)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+
+#define GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST        (GLUSTERD_COMP_BASE + 571)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+
+#define GD_MSG_SNAP_PAUSE_TIER_FAIL                 (GLUSTERD_COMP_BASE + 572)
+
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_SNAP_RESUME_TIER_FAIL                (GLUSTERD_COMP_BASE + 573)
+
 /*------------*/
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index a957377b1a6..19c1f439704 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -1953,6 +1953,142 @@ out:
 }
 
 int
+glusterd_snapshot_pause_tier (xlator_t *this, glusterd_volinfo_t *volinfo)
+{
+        int                     ret             = -1;
+        dict_t                 *dict            = NULL;
+        char                   *op_errstr       = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+
+        if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
+                ret = 0;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                goto out;
+        }
+
+        ret = dict_set_int32 (dict, "rebalance-command",
+                              GF_DEFRAG_CMD_PAUSE_TIER);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_DICT_SET_FAILED,
+                        "Failed to set rebalance-command");
+                goto out;
+        }
+
+        ret = dict_set_str (dict, "volname", volinfo->volname);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_DICT_SET_FAILED,
+                        "Failed to set volname");
+                goto out;
+        }
+
+        ret = gd_brick_op_phase (GD_OP_DEFRAG_BRICK_VOLUME, NULL,
+                                 dict, &op_errstr);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_SNAP_PAUSE_TIER_FAIL,
+                        "Failed to pause tier. Errstr=%s",
+                        op_errstr);
+                goto out;
+        }
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+int
+glusterd_snapshot_resume_tier (xlator_t *this, dict_t *snap_dict)
+{
+        int                     ret             = -1;
+        dict_t                 *dict            = NULL;
+        int64_t                 volcount        = 0;
+        char                    key[PATH_MAX]   = "";
+        char                   *volname         = NULL;
+        int                     i               = 0;
+        char                   *op_errstr       = NULL;
+        glusterd_volinfo_t     *volinfo         = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, snap_dict, out);
+
+        ret = dict_get_int64 (snap_dict, "volcount", &volcount);
+        if (ret) {
+                goto out;
+        }
+        if (volcount <= 0) {
+                ret = -1;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        for (i = 1; i <= volcount; i++) {
+                snprintf (key, sizeof (key), "volname%d", i);
+                ret = dict_get_str (snap_dict, key, &volname);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_SET_FAILED,
+                                "Failed to get key %s", volname);
+                        goto out;
+                }
+
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret)
+                        goto out;
+
+                if (volinfo->type != GF_CLUSTER_TYPE_TIER)
+                        continue;
+
+                ret = dict_set_int32 (dict, "rebalance-command",
+                                      GF_DEFRAG_CMD_RESUME_TIER);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_SET_FAILED,
+                        "Failed to set rebalance-command");
+
+                        goto out;
+                }
+
+                ret = dict_set_str (dict, "volname", volname);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_DICT_SET_FAILED,
+                                "Failed to set volname");
+                        goto out;
+                }
+
+                ret = gd_brick_op_phase (GD_OP_DEFRAG_BRICK_VOLUME, NULL,
+                                         dict, &op_errstr);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_SNAP_RESUME_TIER_FAIL,
+                                "Failed to resume tier");
+                        goto out;
+                }
+        }
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+
+int
 glusterd_snap_create_clone_common_prevalidate (dict_t *rsp_dict, int flags,
                                                char *snapname, char *err_str,
                                                char *snap_volname,
@@ -2254,7 +2390,12 @@ glusterd_snapshot_clone_prevalidate (dict_t *dict, char **op_errstr,
                 goto out;
         }
 
-        ret = 0;
+        ret = glusterd_snapshot_pause_tier (this, snap_vol);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_SNAP_PAUSE_TIER_FAIL,
+                        "Failed to pause tier in clone prevalidate.");
+        }
 out:
 
         if (ret && err_str[0] != '\0') {
@@ -2444,6 +2585,14 @@ glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
                         goto out;
                 }
 
+                ret = glusterd_snapshot_pause_tier (this, volinfo);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_SNAP_PAUSE_TIER_FAIL,
+                                "Failed to pause tier in snap prevalidate.");
+                        goto out;
+                }
+
         }
 
         ret = dict_set_int64 (rsp_dict, "volcount", volcount);
@@ -2454,6 +2603,7 @@ glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
         }
 
         ret = 0;
+
 out:
         if (ret && err_str[0] != '\0') {
                 gf_msg (this->name, loglevel, 0,
@@ -7818,7 +7968,12 @@ glusterd_snapshot_clone_postvalidate (dict_t *dict, int32_t op_ret,
         }
         snap_vol->snapshot = NULL;
 
-        ret = 0;
+        ret = glusterd_snapshot_resume_tier (this, dict);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_SNAP_RESUME_TIER_FAIL,
+                        "Failed to resume tier in clone postvalidate.");
+        }
 
 out:
         return ret;
@@ -7913,7 +8068,13 @@ glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
                 //ignore the errors of autodelete
                 ret = glusterd_handle_snap_limit (dict, rsp_dict);
         }
-        ret = 0;
+
+        ret = glusterd_snapshot_resume_tier (this, dict);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0,
+                        GD_MSG_SNAP_RESUME_TIER_FAIL,
+                        "Failed to resume tier in snapshot postvalidate.");
+        }
 
 out:
         return ret;
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index e16fa0f1b06..7ab873480bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -52,6 +52,15 @@ validate_tier (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
                 goto out;
         }
 
+        else if (strstr (key, "tier-pause")) {
+                if (strcmp(value, "off") &&
+                    strcmp(value, "on")) {
+                        ret = -1;
+                        goto out;
+                }
+                goto out;
+        }
+
         /*
          * Rest of the volume set options for tier are expecting a positive
          * Integer. Change the function accordingly if this constraint is
@@ -1964,6 +1973,13 @@ struct volopt_map_entry glusterd_volopt_map[] = {
                          "file that has read hits less than this value will be "
                          "considered as COLD and will be demoted."
         },
+        { .key         = "cluster.tier-pause",
+          .voltype     = "cluster/tier",
+          .option      = "tier-pause",
+          .op_version  = GD_OP_VERSION_3_7_6,
+          .flags       = OPT_FLAG_CLIENT_OPT,
+          .validate_fn = validate_tier,
+        },
         { .key         = "cluster.tier-promote-frequency",
           .voltype     = "cluster/tier",
           .value       = "120",