gluster rebalance: bring in a 'force' option

* also correct the free space available logic to check the size
  without the file in migration (this considers the sparse files too)

* 'force' option will bypass the free-space check logic, hence will
  cleanup all the linkfile

* 'force' option is valid only with 'migrate-data' option

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2258 (enhance gluster volume rebalance)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2258

4 files changed, 91 insertions, 61 deletions

diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 68ac79878db..7027180f0f5 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -481,7 +481,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (!dict)
                 goto out;
 
-        if (!((wordcount == 4) || (wordcount == 5))) {
+        if (!((wordcount == 4) || (wordcount == 5) || (wordcount == 6))) {
                 cli_usage_out (word->pattern);
                 parse_error = 1;
                 goto out;
@@ -490,7 +490,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
         if (wordcount == 4) {
                 index = 3;
         } else {
-                if (strcmp (words[3], "fix-layout") && 
+                if (strcmp (words[3], "fix-layout") &&
                     strcmp (words[3], "migrate-data")) {
                         cli_usage_out (word->pattern);
                         parse_error = 1;
@@ -499,7 +499,7 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
                 index = 4;
         }
 
-	if (strcmp (words[index], "start") && strcmp (words[index], "stop") && 
+	if (strcmp (words[index], "start") && strcmp (words[index], "stop") &&
             strcmp (words[index], "status")) {
 	        cli_usage_out (word->pattern);
 		parse_error = 1;
@@ -524,6 +524,23 @@ cli_cmd_volume_defrag_cbk (struct cli_state *state, struct cli_cmd_word *word,
                         goto out;
         }
 
+        /* 'force' option is valid only for the 'migrate-data' key */
+        if (wordcount == 6) {
+                if (strcmp (words[3], "migrate-data") ||
+                    strcmp (words[4], "start") ||
+                    strcmp (words[5], "force")) {
+                        cli_usage_out (word->pattern);
+                        parse_error = 1;
+                        goto out;
+                }
+                ret = dict_set_str (dict, "start-type", "migrate-data-force");
+                if (ret)
+                        goto out;
+                ret = dict_set_str (dict, "command", (char *)words[4]);
+                if (ret)
+                        goto out;
+        }
+
         proc = &cli_rpc_prog->proctable[GLUSTER_CLI_DEFRAG_VOLUME];
 
         if (proc->fn) {
@@ -1227,7 +1244,7 @@ struct cli_cmd volume_cmds[] = {
           cli_cmd_volume_remove_brick_cbk,
           "remove brick from volume <VOLNAME>"},
 
-        { "volume rebalance <VOLNAME> [fix-layout|migrate-data] {start|stop|status}",
+        { "volume rebalance <VOLNAME> [fix-layout|migrate-data] {start|stop|status} [force]",
           cli_cmd_volume_defrag_cbk,
           "rebalance operations"},
 
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index 366d8742bc6..8fc73032a33 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -791,6 +791,7 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
         cli_local_t             *local   = NULL;
         char                    *volname = NULL;
         call_frame_t            *frame   = NULL;
+        char                    *status  = "unknown";
         int                      cmd     = 0;
         int                      ret     = 0;
 
@@ -813,16 +814,7 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                 volname = local->u.defrag_vol.volname;
                 cmd = local->u.defrag_vol.cmd;
         }
-        if ((cmd == GF_DEFRAG_CMD_START) ||
-            (cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX) ||
-            (cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA)) {
-                if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
-                        cli_out ("%s", rsp.op_errstr);
-                else
-                        cli_out ("starting rebalance on volume %s has been %s",
-                                 volname, (rsp.op_ret) ? "unsuccessful":
-                                 "successful");
-        }
+
         if (cmd == GF_DEFRAG_CMD_STOP) {
                 if (rsp.op_ret == -1) {
                         if (strcmp (rsp.op_errstr, ""))
@@ -835,6 +827,7 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                                  "(after rebalancing %"PRId64" files totaling "
                                  "%"PRId64" bytes)", volname, rsp.files, rsp.size);
                 }
+                goto done;
         }
         if (cmd == GF_DEFRAG_CMD_STATUS) {
                 if (rsp.op_ret == -1) {
@@ -843,47 +836,55 @@ gf_cli3_1_defrag_volume_cbk (struct rpc_req *req, struct iovec *iov,
                         else
                                 cli_out ("failed to get the status of "
                                          "rebalance process");
-                } else {
-                        char *status = "unknown";
-                        if (rsp.op_errno == 0)
-                                status = "not started";
-                        if (rsp.op_errno == 1)
-                                status = "step 1: layout fix in progress";
-                        if (rsp.op_errno == 2)
-                                status = "step 2: data migration in progress";
-                        if (rsp.op_errno == 3)
-                                status = "stopped";
-                        if (rsp.op_errno == 4)
-                                status = "completed";
-                        if (rsp.op_errno == 5)
-                                status = "failed";
-                        if (rsp.op_errno == 6)
-                                status = "step 1: layout fix complete";
-                        if (rsp.op_errno == 7)
-                                status = "step 2: data migration complete";
-
-                        if (rsp.files && (rsp.op_errno == 1)) {
-                                cli_out ("rebalance %s: fixed layout %"PRId64,
-                                         status, rsp.files);
-                                goto done;
-                        }
-                        if (rsp.files && (rsp.op_errno == 6)) {
-                                cli_out ("rebalance %s: fixed layout %"PRId64,
-                                         status, rsp.files);
-                                goto done;
-                        }
-                        if (rsp.files) {
-                                cli_out ("rebalance %s: rebalanced %"PRId64
-                                         " files of size %"PRId64" (total files"
-                                         " scanned %"PRId64")", status,
-                                         rsp.files, rsp.size, rsp.lookedup_files);
-                                goto done;
-                        }
-
-                        cli_out ("rebalance %s", status);
+                        goto done;
+                }
+                if (rsp.op_errno == 0)
+                        status = "not started";
+                if (rsp.op_errno == 1)
+                        status = "step 1: layout fix in progress";
+                if (rsp.op_errno == 2)
+                        status = "step 2: data migration in progress";
+                if (rsp.op_errno == 3)
+                        status = "stopped";
+                if (rsp.op_errno == 4)
+                        status = "completed";
+                if (rsp.op_errno == 5)
+                        status = "failed";
+                if (rsp.op_errno == 6)
+                        status = "step 1: layout fix complete";
+                if (rsp.op_errno == 7)
+                        status = "step 2: data migration complete";
+
+                if (rsp.files && (rsp.op_errno == 1)) {
+                        cli_out ("rebalance %s: fixed layout %"PRId64,
+                                 status, rsp.files);
+                        goto done;
                 }
+                if (rsp.files && (rsp.op_errno == 6)) {
+                        cli_out ("rebalance %s: fixed layout %"PRId64,
+                                 status, rsp.files);
+                        goto done;
+                }
+                if (rsp.files) {
+                        cli_out ("rebalance %s: rebalanced %"PRId64
+                                 " files of size %"PRId64" (total files"
+                                 " scanned %"PRId64")", status,
+                                 rsp.files, rsp.size, rsp.lookedup_files);
+                        goto done;
+                }
+
+                cli_out ("rebalance %s", status);
+                goto done;
         }
 
+        /* All other possibility is about starting a volume */
+        if (rsp.op_ret && strcmp (rsp.op_errstr, ""))
+                cli_out ("%s", rsp.op_errstr);
+        else
+                cli_out ("starting rebalance on volume %s has been %s",
+                         volname, (rsp.op_ret) ? "unsuccessful":
+                         "successful");
+
 done:
         if (volname)
                 GF_FREE (volname);
@@ -1906,6 +1907,9 @@ gf_cli3_1_defrag_volume (call_frame_t *frame, xlator_t *this,
                         if (strcmp (cmd_str, "migrate-data") == 0) {
                                 req.cmd = GF_DEFRAG_CMD_START_MIGRATE_DATA;
                         }
+                        if (strcmp (cmd_str, "migrate-data-force") == 0) {
+                                req.cmd = GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE;
+                        }
                 }
                 goto done;
         }
diff --git a/rpc/xdr/src/cli1.h b/rpc/xdr/src/cli1.h
index 26b142031a8..2aefe0737e5 100644
--- a/rpc/xdr/src/cli1.h
+++ b/rpc/xdr/src/cli1.h
@@ -31,6 +31,7 @@ enum gf_cli_defrag_type {
         GF_DEFRAG_CMD_STATUS,
         GF_DEFRAG_CMD_START_LAYOUT_FIX,
         GF_DEFRAG_CMD_START_MIGRATE_DATA,
+        GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE,
 };
 
 ssize_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index 75c175b16b8..3aba3b99076 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -43,6 +43,8 @@
 #include "syscall.h"
 #include "cli1.h"
 
+#define GF_DISK_SECTOR_SIZE 512
+
 int
 gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
 {
@@ -117,18 +119,22 @@ gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
 
                 /* Prevent data movement from a node which has higher
                    disk-space to a node with lesser */
-                {
+                if (defrag->cmd != GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE) {
                         ret = statvfs (full_path, &src_statfs);
                         if (ret)
-                                gf_log ("", GF_LOG_INFO, "statfs on %s failed",
-                                        full_path);
+                                gf_log ("", GF_LOG_WARNING,
+                                        "statfs on %s failed", full_path);
 
                         ret = statvfs (tmp_filename, &dst_statfs);
                         if (ret)
-                                gf_log ("", GF_LOG_INFO, "statfs on %s failed",
-                                        tmp_filename);
-
-                        if (dst_statfs.f_bavail < src_statfs.f_bavail) {
+                                gf_log ("", GF_LOG_WARNING,
+                                        "statfs on %s failed", tmp_filename);
+
+                        /* Calculate the size without the file in migration */
+                        if (((dst_statfs.f_bavail *
+                              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) >
+                            (((src_statfs.f_bavail * src_statfs.f_bsize) /
+                              GF_DISK_SECTOR_SIZE) - stbuf.st_blocks)) {
                                 gf_log ("", GF_LOG_INFO,
                                         "data movement attempted from node with"
                                         " higher disk space to a node with "
@@ -337,8 +343,7 @@ glusterd_defrag_start (void *data)
                 volinfo->defrag_status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
         }
 
-        if ((defrag->cmd == GF_DEFRAG_CMD_START) ||
-            (defrag->cmd == GF_DEFRAG_CMD_START_MIGRATE_DATA)) {
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
                 /* It was used by number of layout fixes on directories */
                 defrag->total_files = 0;
 
@@ -490,6 +495,7 @@ glusterd_rebalance_cmd_attempted_log (int cmd, char *volname)
                                     volname);
                         break;
                 case GF_DEFRAG_CMD_START_MIGRATE_DATA:
+                case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
                         gf_cmd_log ("Volume rebalance"," on volname: %s "
                                     "cmd: start data migrate attempted",
                                     volname);
@@ -683,6 +689,7 @@ glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req)
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA:
+        case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
                 ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
                                                     cli_req.cmd);
                 rsp.op_ret = ret;
@@ -743,6 +750,7 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_MIGRATE_DATA:
+        case GF_DEFRAG_CMD_START_MIGRATE_DATA_FORCE:
         {
                 ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
                                                     cli_req.cmd);