gluster rebalance: get the proper/exact error msg to cli

introduce a new field in XDR structure, and hence change the version
of program.

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 1922 (Volume not present wrong message displayed on command line)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1922

3 files changed, 314 insertions, 84 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index efba9e202c7..68cdb6fc3a7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -3532,7 +3532,7 @@ struct rpcsvc_program gd_svc_mgmt_prog = {
 rpcsvc_actor_t gd_svc_cli_actors[] = {
         [GLUSTER_CLI_PROBE]         = { "CLI_PROBE", GLUSTER_CLI_PROBE, glusterd_handle_cli_probe, NULL, NULL},
         [GLUSTER_CLI_CREATE_VOLUME] = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME, glusterd_handle_create_volume, NULL,NULL},
-        [GLUSTER_CLI_DEFRAG_VOLUME] = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME, glusterd_handle_defrag_volume, NULL,NULL},
+        [GLUSTER_CLI_DEFRAG_VOLUME] = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME, glusterd_handle_defrag_volume_v2, NULL,NULL},
         [GLUSTER_CLI_DEPROBE]       = { "FRIEND_REMOVE", GLUSTER_CLI_DEPROBE, glusterd_handle_cli_deprobe, NULL, NULL},
         [GLUSTER_CLI_LIST_FRIENDS]  = { "LIST_FRIENDS", GLUSTER_CLI_LIST_FRIENDS, glusterd_handle_cli_list_friends, NULL, NULL},
         [GLUSTER_CLI_START_VOLUME]  = { "START_VOLUME", GLUSTER_CLI_START_VOLUME, glusterd_handle_cli_start_volume, NULL, NULL},
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index a8a063706fb..c1dbead2eed 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -108,25 +108,27 @@ gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
                 while (1) {
                         ret = read (src_fd, defrag->databuf, 131072);
                         if (!ret || (ret < 0)) {
-                                close (dst_fd);
-                                close (src_fd);
                                 break;
                         }
                         ret = write (dst_fd, defrag->databuf, ret);
                         if (ret < 0) {
-                                close (dst_fd);
-                                close (src_fd);
                                 break;
                         }
                 }
 
                 ret = stat (full_path, &new_stbuf);
-                if (ret < 0)
+                if (ret < 0) {
+                        close (dst_fd);
+                        close (src_fd);
                         continue;
+                }
                 /* No need to rebalance, if there is some
                    activity on source file */
-                if (new_stbuf.st_mtime != stbuf.st_mtime)
+                if (new_stbuf.st_mtime != stbuf.st_mtime) {
+                        close (dst_fd);
+                        close (src_fd);
                         continue;
+                }
 
                 ret = fchmod (dst_fd, stbuf.st_mode);
                 if (ret) {
@@ -152,6 +154,9 @@ gf_glusterd_rebalance_move_data (glusterd_volinfo_t *volinfo, const char *dir)
                         UNLOCK (&defrag->lock);
                 }
 
+                close (dst_fd);
+                close (src_fd);
+
                 if (volinfo->defrag_status == GF_DEFRAG_STATUS_STOPED) {
                         closedir (fd);
                         ret = -1;
@@ -329,22 +334,77 @@ out:
 }
 
 int
-glusterd_defrag_stop (glusterd_volinfo_t *volinfo,
-                      gf1_cli_defrag_vol_rsp *rsp)
+glusterd_defrag_stop_validate (glusterd_volinfo_t *volinfo,
+                               char *op_errstr, size_t len)
+{
+        int     ret = -1;
+        if (glusterd_is_defrag_on (volinfo) == 0) {
+                snprintf (op_errstr, len, "Rebalance on %s is either Completed "
+                          "or not yet started", volinfo->volname);
+                goto out;
+        }
+        ret = 0;
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_defrag_stop (glusterd_volinfo_t *volinfo, u_quad_t *files,
+                      u_quad_t *size, char *op_errstr, size_t len)
 {
         /* TODO: set a variaeble 'stop_defrag' here, it should be checked
            in defrag loop */
-        if (!volinfo || !volinfo->defrag)
+        int     ret = -1;
+        GF_ASSERT (volinfo);
+        GF_ASSERT (files);
+        GF_ASSERT (size);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_defrag_stop_validate (volinfo, op_errstr, len);
+        if (ret)
+                goto out;
+        if (!volinfo || !volinfo->defrag) {
+                ret = -1;
                 goto out;
+        }
 
         LOCK (&volinfo->defrag->lock);
         {
                 volinfo->defrag_status = GF_DEFRAG_STATUS_STOPED;
-                rsp->files = volinfo->defrag->total_files;
-                rsp->size = volinfo->defrag->total_data;
+                *files = volinfo->defrag->total_files;
+                *size = volinfo->defrag->total_data;
         }
         UNLOCK (&volinfo->defrag->lock);
 
+        ret = 0;
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_defrag_status_get_v2 (glusterd_volinfo_t *volinfo,
+                            gf2_cli_defrag_vol_rsp *rsp)
+{
+        if (!volinfo)
+                goto out;
+
+        if (volinfo->defrag) {
+                LOCK (&volinfo->defrag->lock);
+                {
+                        rsp->files = volinfo->defrag->total_files;
+                        rsp->size = volinfo->defrag->total_data;
+                        rsp->lookedup_files = volinfo->defrag->num_files_lookedup;
+                }
+                UNLOCK (&volinfo->defrag->lock);
+        } else {
+                rsp->files = volinfo->rebalance_files;
+                rsp->size  = volinfo->rebalance_data;
+                rsp->lookedup_files = volinfo->lookedup_files;
+        }
+
+        rsp->op_errno = volinfo->defrag_status;
         rsp->op_ret = 0;
 out:
         return 0;
@@ -377,16 +437,167 @@ out:
         return 0;
 }
 
+void
+glusterd_rebalance_cmd_attempted_log (int cmd, char *volname)
+{
+        switch (cmd) {
+                case GF_DEFRAG_CMD_START:
+                        gf_cmd_log ("Volume rebalance"," on volname: %s "
+                                    "cmd: start, attempted", volname);
+                        break;
+                case GF_DEFRAG_CMD_STOP:
+                        gf_cmd_log ("Volume rebalance"," on volname: %s "
+                                    "cmd: stop, attempted", volname);
+                        break;
+                default:
+                        break;
+        }
+        gf_log ("glusterd", GF_LOG_NORMAL, "Received rebalance volume %s on %s",
+                (cmd == GF_DEFRAG_CMD_START)?"start":(cmd == GF_DEFRAG_CMD_STOP)?"stop":"status"
+                , volname);
+}
+
+void
+glusterd_rebalance_cmd_log (int cmd, char *volname, int status)
+{
+        if (cmd != GF_DEFRAG_CMD_STATUS) {
+                gf_cmd_log ("volume rebalance"," on volname: %s %d %s",
+                            volname, cmd, ((status)?"FAILED":"SUCCESS"));
+        }
+}
+
 int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
+                                size_t len)
 {
-        int32_t                ret           = -1;
-        gf1_cli_defrag_vol_req cli_req       = {0,};
-        glusterd_conf_t         *priv = NULL;
-        char                   cmd_str[4096] = {0,};
-        glusterd_volinfo_t      *volinfo = NULL;
+        int     ret = -1;
+
+        if (glusterd_is_defrag_on (volinfo)) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "rebalance on volume %s already started",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Rebalance on %s is already started",
+                          volinfo->volname);
+                goto out;
+        }
+
+        if (glusterd_is_rb_started (volinfo) ||
+            glusterd_is_rb_paused (volinfo)) {
+                gf_log ("glusterd", GF_LOG_DEBUG,
+                        "Replace brick is in progress on volume %s",
+                        volinfo->volname);
+                snprintf (op_errstr, len, "Replace brick is in progress on "
+                          "volume %s", volinfo->volname);
+                goto out;
+        }
+        ret = 0;
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
+                              size_t len)
+{
+        int                    ret = -1;
         glusterd_defrag_info_t *defrag =  NULL;
-        gf1_cli_defrag_vol_rsp rsp = {0,};
+        char                   cmd_str[4096] = {0,};
+        glusterd_conf_t        *priv = NULL;
+
+        priv    = THIS->private;
+
+        GF_ASSERT (volinfo);
+        GF_ASSERT (op_errstr);
+
+        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len);
+        if (ret)
+                goto out;
+        if (!volinfo->defrag)
+                volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t),
+                                             gf_gld_mt_defrag_info);
+        if (!volinfo->defrag)
+                goto out;
+
+        defrag = volinfo->defrag;
+
+        LOCK_INIT (&defrag->lock);
+        snprintf (defrag->mount, 1024, "%s/mount/%s",
+                  priv->workdir, volinfo->volname);
+        /* Create a directory, mount glusterfs over it, start glusterfs-defrag */
+        snprintf (cmd_str, sizeof (cmd_str), "mkdir -p %s", defrag->mount);
+        ret = system (cmd_str);
+
+        if (ret) {
+                gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed", cmd_str);
+                goto out;
+        }
+
+        snprintf (cmd_str, sizeof (cmd_str), "%s/sbin/glusterfs -s localhost "
+                  "--volfile-id %s --volume-name %s-quick-read "
+                  "--xlator-option *dht.unhashed-sticky-bit=yes "
+                  "--xlator-option *dht.use-readdirp=yes "
+                  "--xlator-option *dht.lookup-unhashed=yes %s",
+                  GFS_PREFIX, volinfo->volname, volinfo->volname,
+                  defrag->mount);
+        ret = gf_system (cmd_str);
+        if (ret) {
+                gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed", cmd_str);
+                goto out;
+        }
+
+        volinfo->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+        ret = pthread_create (&defrag->th, NULL, glusterd_defrag_start,
+                              volinfo);
+        if (ret) {
+                snprintf (cmd_str, sizeof (cmd_str), "umount -l %s", defrag->mount);
+                if (system (cmd_str))
+                        gf_log("glusterd", GF_LOG_DEBUG, "command: %s "
+                               "failed", cmd_str);
+        }
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_rebalance_cmd_validate (int cmd, char *volname,
+                                 glusterd_volinfo_t **volinfo,
+                                 char *op_errstr, size_t len)
+{
+        int ret = -1;
+
+        if (glusterd_volinfo_find(volname, volinfo)) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on invalid"
+                        " volname %s", volname);
+                snprintf (op_errstr, len, "Volume %s does not exist",
+                          volname);
+                goto out;
+        }
+
+        if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
+                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on stopped"
+                        " volname %s", volname);
+                snprintf (op_errstr, len, "Volume %s needs to "
+                          "be started to perform rebalance", volname);
+                goto out;
+        }
+        ret = 0;
+out:
+        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
+
+int
+glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req)
+{
+        int32_t                 ret           = -1;
+        gf1_cli_defrag_vol_req  cli_req       = {0,};
+        glusterd_volinfo_t     *volinfo = NULL;
+        gf2_cli_defrag_vol_rsp rsp = {0,};
+        char                    msg[2048] = {0};
+        glusterd_conf_t        *priv = NULL;
 
         GF_ASSERT (req);
 
@@ -409,88 +620,105 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
                 default:
                         break;
         }
+
         gf_log ("glusterd", GF_LOG_NORMAL, "Received rebalance volume %s on %s",
-                (cli_req.cmd == GF_DEFRAG_CMD_START)?"start":(cli_req.cmd == GF_DEFRAG_CMD_STOP)?"stop":"status"
-                , cli_req.volname);
+                ((cli_req.cmd == GF_DEFRAG_CMD_START) ? "start" :
+                 (cli_req.cmd == GF_DEFRAG_CMD_STOP) ? "stop" : "status"),
+                cli_req.volname);
+
+        glusterd_rebalance_cmd_attempted_log (cli_req.cmd, cli_req.volname);
 
         rsp.volname = cli_req.volname;
         rsp.op_ret = -1;
-        if (glusterd_volinfo_find(cli_req.volname, &volinfo)) {
-                gf_log ("glusterd", GF_LOG_NORMAL, "Received rebalance on invalid"
-                        " volname %s", cli_req.volname);
-                goto out;
-        }
+        rsp.op_errstr = msg;
 
-        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
-                gf_log ("glusterd", GF_LOG_NORMAL, "Received rebalance on stopped"
-                        " volname %s", cli_req.volname);
+        ret = glusterd_rebalance_cmd_validate (cli_req.cmd, cli_req.volname,
+                                               &volinfo, msg, sizeof (msg));
+        if (ret)
                 goto out;
-        }
-
         switch (cli_req.cmd) {
         case GF_DEFRAG_CMD_START:
-        {
-                if (volinfo->defrag) {
-                        gf_log ("glusterd", GF_LOG_DEBUG,
-                                "rebalance on volume %s already started",
-                                cli_req.volname);
-                        rsp.op_errno = EEXIST;
-                        goto out;
-                }
+                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg));
+                rsp.op_ret = ret;
+                break;
+        case GF_DEFRAG_CMD_STOP:
+                ret = glusterd_defrag_stop (volinfo, &rsp.files, &rsp.size,
+                                            msg, sizeof (msg));
+                rsp.op_ret = ret;
+                break;
+        case GF_DEFRAG_CMD_STATUS:
+                ret = glusterd_defrag_status_get_v2 (volinfo, &rsp);
+                break;
+        default:
+                break;
+        }
+        glusterd_rebalance_cmd_log (cli_req.cmd, cli_req.volname, rsp.op_ret);
+out:
 
-                if (glusterd_is_rb_started (volinfo) ||
-                    glusterd_is_rb_paused (volinfo)) {
-                        gf_log ("glusterd", GF_LOG_DEBUG,
-                                "Replace brick is in progress on volume %s",
-                                cli_req.volname);
-                        goto out;
-                }
-                volinfo->defrag = GF_CALLOC (1, sizeof (glusterd_defrag_info_t),
-                                             gf_gld_mt_defrag_info);
-                if (!volinfo->defrag)
-                        goto out;
+        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
+                                     gf_xdr_serialize_cli_defrag_vol_rsp_v2);
+        if (cli_req.volname)
+                free (cli_req.volname);//malloced by xdr
 
-                defrag = volinfo->defrag;
+        return 0;
+}
 
-                LOCK_INIT (&defrag->lock);
-                snprintf (defrag->mount, 1024, "%s/mount/%s",
-                          priv->workdir, cli_req.volname);
-                /* Create a directory, mount glusterfs over it, start glusterfs-defrag */
-                snprintf (cmd_str, 4096, "mkdir -p %s", defrag->mount);
-                ret = system (cmd_str);
+int
+glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+{
+        int32_t                ret           = -1;
+        gf1_cli_defrag_vol_req cli_req       = {0,};
+        glusterd_conf_t         *priv = NULL;
+        char                   cmd_str[4096] = {0,};
+        glusterd_volinfo_t      *volinfo = NULL;
+        gf1_cli_defrag_vol_rsp rsp = {0,};
+        char                    msg[2048] = {0};
 
-                if (ret) {
-                        gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed", cmd_str);
-                        goto out;
-                }
+        GF_ASSERT (req);
 
-                snprintf (cmd_str, 4096, "%s/sbin/glusterfs -s localhost "
-                          "--volfile-id %s --volume-name %s-quick-read "
-                          "--xlator-option *dht.unhashed-sticky-bit=yes "
-                          "--xlator-option *dht.use-readdirp=yes "
-                          "--xlator-option *dht.lookup-unhashed=yes %s",
-                          GFS_PREFIX, cli_req.volname, cli_req.volname,
-                          defrag->mount);
-                ret = gf_system (cmd_str);
-                if (ret) {
-                        gf_log("glusterd", GF_LOG_DEBUG, "command: %s failed", cmd_str);
-                        goto out;
-                }
+        priv    = THIS->private;
 
-                volinfo->defrag_status = GF_DEFRAG_STATUS_STARTED;
-                rsp.op_ret = 0;
+        if (!gf_xdr_to_cli_defrag_vol_req (req->msg[0], &cli_req)) {
+                //failed to decode msg;
+                req->rpc_err = GARBAGE_ARGS;
+                goto out;
+        }
 
-                ret = pthread_create (&defrag->th, NULL, glusterd_defrag_start,
-                                      volinfo);
-                if (ret) {
-                        snprintf (cmd_str, 1024, "umount -l %s", defrag->mount);
-                        ret = system (cmd_str);
-                        rsp.op_ret = -1;
-                }
+        switch (cli_req.cmd) {
+                case GF_DEFRAG_CMD_START:
+                        gf_cmd_log ("Volume rebalance"," on volname: %s "
+                                    "cmd: start, attempted", cli_req.volname);
+                        break;
+                case GF_DEFRAG_CMD_STOP:
+                        gf_cmd_log ("Volume rebalance"," on volname: %s "
+                                    "cmd: stop, attempted", cli_req.volname);
+                        break;
+                default:
+                        break;
+        }
+        gf_log ("glusterd", GF_LOG_NORMAL, "Received rebalance volume %s on %s",
+                ((cli_req.cmd == GF_DEFRAG_CMD_START) ? "start" :
+                 (cli_req.cmd == GF_DEFRAG_CMD_STOP) ? "stop" : "status"),
+                cli_req.volname);
+
+        rsp.volname = cli_req.volname;
+        rsp.op_ret = -1;
+
+        ret = glusterd_rebalance_cmd_validate (cli_req.cmd, cli_req.volname,
+                                               &volinfo, msg, sizeof (msg));
+        if (ret)
+                goto out;
+        switch (cli_req.cmd) {
+        case GF_DEFRAG_CMD_START:
+        {
+                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg));
+                rsp.op_ret = ret;
                 break;
         }
         case GF_DEFRAG_CMD_STOP:
-                ret = glusterd_defrag_stop (volinfo, &rsp);
+                ret = glusterd_defrag_stop (volinfo, &rsp.files, &rsp.size,
+                                            msg, sizeof (msg));
+                rsp.op_ret = ret;
                 break;
         case GF_DEFRAG_CMD_STATUS:
                 ret = glusterd_defrag_status_get (volinfo, &rsp);
@@ -508,7 +736,6 @@ glusterd_handle_defrag_volume (rpcsvc_request_t *req)
         }
 
 out:
-
         ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
                                      gf_xdr_serialize_cli_defrag_vol_rsp);
         if (cli_req.volname)
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index d98ddb1c6b7..06f35479c35 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -354,6 +354,9 @@ int
 glusterd_handle_defrag_volume (rpcsvc_request_t *req);
 
 int
+glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
+
+int
 glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
                               int32_t op_errno, char *hostname, int port);