glusterd: Make reset-brick work correctly if brick-mux is on

Reset brick currently kills of the corresponding brick process.
However, with brick multiplexing enabled, stopping the brick
process would render all bricks attached to it unavailable. To
handle this correctly, we need to make sure that the brick process
is terminated only if brick-multiplexing is disabled. Otherwise,
we should send the GLUSTERD_BRICK_TERMINATE rpc to the respective
brick process to detach the brick that is to be reset.

> Signed-off-by: Samikshan Bairagya <samikshan@gmail.com>
> Reviewed-on: https://review.gluster.org/17128
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
> Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

(cherry picked from commit 74383e3ec6f8244b3de9bf14016452498c1ddcf0)

Change-Id: I69002d66ffe6ec36ef48af09b66c522c6d35ac58
BUG: 1449934
Signed-off-by: Samikshan Bairagya <samikshan@gmail.com>
Reviewed-on: https://review.gluster.org/17253
Smoke: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

14 files changed, 194 insertions, 85 deletions

diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index 5f07a057d47..16f33c5161e 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -2087,6 +2087,7 @@ cli_cmd_volume_reset_brick_cbk (struct cli_state *state,
         if (!frame)
                 goto out;
 
+
         ret = cli_cmd_volume_reset_brick_parse (words, wordcount, &options);
 
         if (ret) {
@@ -2095,6 +2096,15 @@ cli_cmd_volume_reset_brick_cbk (struct cli_state *state,
                 goto out;
         }
 
+        if (state->mode & GLUSTER_MODE_WIGNORE_PARTITION) {
+                ret = dict_set_int32 (options, "ignore-partition", _gf_true);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR, "Failed to set ignore-"
+                                "partition option");
+                        goto out;
+                }
+        }
+
         CLI_LOCAL_INIT (local, words, frame, options);
 
         if (proc->fn) {
diff --git a/cli/src/cli.c b/cli/src/cli.c
index c9c578f253c..1cd9eb79fe9 100644
--- a/cli/src/cli.c
+++ b/cli/src/cli.c
@@ -348,6 +348,11 @@ cli_opt_parse (char *opt, struct cli_state *state)
                 return 0;
         }
 
+        if (strcmp (opt, "wignore-partition") == 0) {
+                state->mode |= GLUSTER_MODE_WIGNORE_PARTITION;
+                return 0;
+        }
+
         if (strcmp (opt, "wignore") == 0) {
                 state->mode |= GLUSTER_MODE_WIGNORE;
                 return 0;
diff --git a/cli/src/cli.h b/cli/src/cli.h
index 27a3f3cc136..ef632d8a711 100644
--- a/cli/src/cli.h
+++ b/cli/src/cli.h
@@ -56,10 +56,11 @@ typedef enum {
         MAX
 } values;
 
-#define GLUSTER_MODE_SCRIPT    (1 << 0)
-#define GLUSTER_MODE_ERR_FATAL (1 << 1)
-#define GLUSTER_MODE_XML       (1 << 2)
-#define GLUSTER_MODE_WIGNORE   (1 << 3)
+#define GLUSTER_MODE_SCRIPT             (1 << 0)
+#define GLUSTER_MODE_ERR_FATAL          (1 << 1)
+#define GLUSTER_MODE_XML                (1 << 2)
+#define GLUSTER_MODE_WIGNORE            (1 << 3)
+#define GLUSTER_MODE_WIGNORE_PARTITION  (1 << 4)
 
 
 #define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
diff --git a/tests/bugs/glusterd/bug-1446172-brick-mux-reset-brick.t b/tests/bugs/glusterd/bug-1446172-brick-mux-reset-brick.t
new file mode 100644
index 00000000000..e6aaaa4e87c
--- /dev/null
+++ b/tests/bugs/glusterd/bug-1446172-brick-mux-reset-brick.t
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../traps.rc
+. $(dirname $0)/../../volume.rc
+
+cleanup;
+
+function count_up_bricks {
+        $CLI --xml volume status | grep '<status>1' | wc -l
+}
+
+function count_brick_processes {
+	pgrep glusterfsd | wc -l
+}
+
+TEST glusterd
+
+TEST $CLI volume set all cluster.brick-multiplex on
+push_trapfunc "$CLI volume set all cluster.brick-multiplex off"
+push_trapfunc "cleanup"
+
+TEST $CLI volume create $V0 $H0:$B0/${V0}{0,1}
+TEST $CLI volume create $V1 $H0:$B0/${V1}{0,1}
+
+TEST $CLI volume start $V0
+TEST $CLI volume start $V1
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_up_bricks
+EXPECT 1 count_brick_processes
+
+TEST glusterfs --volfile-id=$V0 --volfile-server=$H0 $M0;
+# Create files
+for i in {1..5}
+do
+        echo $i > $M0/file$i.txt
+done
+
+TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}1 start
+
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 3 count_up_bricks
+EXPECT 1 count_brick_processes
+
+# Negative case with brick killed but volume-id xattr present
+TEST ! $CLI volume reset-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}1 commit
+
+# reset-brick commit force should work and should bring up the brick
+TEST $CLI volume reset-brick $V0 $H0:$B0/${V0}1 $H0:$B0/${V0}1 commit force
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_up_bricks
+EXPECT 1 count_brick_processes
+
+TEST glusterfs --volfile-id=$V1 --volfile-server=$H0 $M1;
+# Create files
+for i in {1..5}
+do
+        echo $i > $M1/file$i.txt
+done
+
+TEST $CLI volume reset-brick $V1 $H0:$B0/${V1}1 start
+
+EXPECT_WITHIN $PROCESS_DOWN_TIMEOUT 3 count_up_bricks
+EXPECT 1 count_brick_processes
+
+# Simulate reset disk
+for i in {1..5}
+do
+        rm -rf $B0/${V1}1/file$i.txt
+done
+
+setfattr -x trusted.glusterfs.volume-id $B0/${V1}1
+setfattr -x trusted.gfid $B0/${V1}1
+
+# Test reset-brick commit. Using CLI_IGNORE_PARTITION since normal CLI  uses
+# the --wignore flag that essentially makes the command act like "commit force"
+TEST $CLI_IGNORE_PARTITION volume reset-brick $V1 $H0:$B0/${V1}1 $H0:$B0/${V1}1 commit
+
+EXPECT_WITHIN $PROCESS_UP_TIMEOUT 4 count_up_bricks
+EXPECT 1 count_brick_processes
diff --git a/tests/include.rc b/tests/include.rc
index 22265755a02..5b800b5b080 100644
--- a/tests/include.rc
+++ b/tests/include.rc
@@ -90,7 +90,12 @@ LOGDIR=$(gluster --print-logdir)
 statedumpdir=`gluster --print-statedumpdir`; # Default directory for statedump
 
 CLI="gluster --mode=script --wignore";
-CLI_NO_FORCE="gluster --mode-script";
+CLI_NO_FORCE="gluster --mode=script";
+
+# CLI_IGNORE_PARTITION makes sure that the warning related to bricks being on
+# root partition is ignored while running the command in a "no force" mode
+CLI_IGNORE_PARTITION="gluster --mode=script --wignore-partition"
+
 _GFS () {
 	glusterfs "$@"
 	local mount_ret=$?
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index a99f148d46f..5bb347cd6d8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -1883,7 +1883,8 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 
                         ret = glusterd_validate_and_create_brickpath (brickinfo,
                                                           volinfo->volume_id,
-                                                          op_errstr, is_force);
+                                                          op_errstr, is_force,
+                                                          _gf_false);
                         if (ret)
                                 goto out;
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index ded5f9b3dc0..555c2178692 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -41,7 +41,7 @@
 
 #define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
 
-#define GLFS_NUM_MESSAGES       598
+#define GLFS_NUM_MESSAGES       599
 
 #define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
 /* Messaged with message IDs */
@@ -4845,6 +4845,14 @@
 
 #define GD_MSG_GANESHA_NOT_RUNNING                 (GLUSTERD_COMP_BASE + 598)
 
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+#define GD_MSG_PIDFILE_UNLINKING                   (GLUSTERD_COMP_BASE + 599)
+
 /*------------*/
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index b4846f2c317..8075fda0c00 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -6157,8 +6157,6 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
         glusterd_volinfo_t                      *volinfo = NULL;
         glusterd_brickinfo_t                    *brickinfo = NULL;
         glusterd_pending_node_t                 *pending_node = NULL;
-        glusterd_conf_t                         *conf = THIS->private;
-        char                                    pidfile[PATH_MAX] = {0,};
 
         ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
         if (ret)
@@ -6194,11 +6192,6 @@ glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
                          */
                         brickinfo->status = GF_BRICK_STOPPED;
                         brickinfo->started_here = _gf_false;
-                        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo,
-                                                    brickinfo, conf);
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "unlinking pidfile %s", pidfile);
-                        (void) sys_unlink (pidfile);
                 }
         }
 
@@ -6221,8 +6214,6 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
         glusterd_pending_node_t                 *pending_node = NULL;
         int32_t                                 command = 0;
         int32_t                                 force = 0;
-        glusterd_conf_t                         *conf = THIS->private;
-        char                                    pidfile[PATH_MAX] = {0,};
 
         ret = dict_get_str (dict, "volname", &volname);
 
@@ -6303,11 +6294,6 @@ glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
                          */
                         brickinfo->status = GF_BRICK_STOPPED;
                         brickinfo->started_here = _gf_false;
-                        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo,
-                                                    brickinfo, conf);
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "unlinking pidfile %s", pidfile);
-                        (void) sys_unlink (pidfile);
                 }
                 i++;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index fb29c6efcfd..887c3496475 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -252,7 +252,8 @@ glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
         if (gf_is_local_addr (host)) {
                 ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
                                                   volinfo->volume_id,
-                                                  op_errstr, is_force);
+                                                  op_errstr, is_force,
+                                                  _gf_false);
                 if (ret)
                         goto out;
         }
diff --git a/xlators/mgmt/glusterd/src/glusterd-reset-brick.c b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
index 2e547c7e09f..c127d64d119 100644
--- a/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
@@ -47,6 +47,7 @@ glusterd_reset_brick_prevalidate (dict_t *dict, char **op_errstr,
         char                                     pidfile[PATH_MAX]  = {0};
         xlator_t                                *this               = NULL;
         gf_boolean_t                             is_force           = _gf_false;
+        int32_t                                  ignore_partition   = 0;
         pid_t                                    pid                = -1;
         uuid_t                                   volume_id          = {0,};
         char                                    *dup_dstbrick       = NULL;
@@ -153,11 +154,14 @@ glusterd_reset_brick_prevalidate (dict_t *dict, char **op_errstr,
         volinfo->rep_brick.src_brick = src_brickinfo;
         volinfo->rep_brick.dst_brick = dst_brickinfo;
 
+        ret = dict_get_int32 (dict, "ignore-partition", &ignore_partition);
+        ret = 0;
         if (gf_is_local_addr (host)) {
                 ret = glusterd_validate_and_create_brickpath
                                                   (dst_brickinfo,
                                                   volinfo->volume_id,
-                                                  op_errstr, is_force);
+                                                  op_errstr, is_force,
+                                                  ignore_partition);
                 if (ret)
                         goto out;
         } else {
@@ -234,17 +238,16 @@ out:
 int
 glusterd_op_reset_brick (dict_t *dict, dict_t *rsp_dict)
 {
-        int                                      ret           = 0;
-        char                                    *op            = NULL;
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        char                                    *volname       = NULL;
-        xlator_t                                *this          = NULL;
-        glusterd_conf_t                         *priv          = NULL;
-        char                                    *src_brick     = NULL;
-        char                                    *dst_brick     = NULL;
-        glusterd_brickinfo_t                    *src_brickinfo = NULL;
-        glusterd_brickinfo_t                    *dst_brickinfo = NULL;
-        char                                    pidfile[PATH_MAX] = {0,};
+        int                              ret           = 0;
+        char                            *op            = NULL;
+        glusterd_volinfo_t              *volinfo       = NULL;
+        char                            *volname       = NULL;
+        xlator_t                        *this          = NULL;
+        glusterd_conf_t                 *priv          = NULL;
+        char                            *src_brick     = NULL;
+        char                            *dst_brick     = NULL;
+        glusterd_brickinfo_t            *src_brickinfo = NULL;
+        glusterd_brickinfo_t            *dst_brickinfo = NULL;
 
         this = THIS;
         GF_ASSERT (this);
@@ -289,26 +292,18 @@ glusterd_op_reset_brick (dict_t *dict, dict_t *rsp_dict)
         }
 
         if (!strcmp (op, "GF_RESET_OP_START")) {
-                (void) glusterd_brick_disconnect (src_brickinfo);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo,
-                                            src_brickinfo, priv);
-                ret = glusterd_service_stop ("brick", pidfile,
-                                             SIGTERM, _gf_false);
-                if (ret == 0) {
-                        glusterd_set_brick_status (src_brickinfo,
-                                                GF_BRICK_STOPPED);
-                        (void) glusterd_brick_unlink_socket_file
-                                        (volinfo, src_brickinfo);
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_BRICK_CLEANUP_SUCCESS,
-                        "Brick cleanup successful.");
-                } else {
+                ret = glusterd_volume_stop_glusterfs (volinfo,
+                                                      src_brickinfo,
+                                                      _gf_false);
+                if (ret) {
                         gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_BRK_CLEANUP_FAIL,
-                                "Unable to cleanup src brick");
-                        goto out;
+                                GD_MSG_BRICK_STOP_FAIL, "Unable to stop"
+                                " brick: %s:%s", src_brickinfo->hostname,
+                                src_brickinfo->path);
                 }
+
                 goto out;
+
         } else if (!strcmp (op, "GF_RESET_OP_COMMIT") ||
                    !strcmp (op, "GF_RESET_OP_COMMIT_FORCE")) {
                 ret = dict_get_str (dict, "dst-brick", &dst_brick);
@@ -344,25 +339,17 @@ glusterd_op_reset_brick (dict_t *dict, dict_t *rsp_dict)
                 if (ret)
                         goto out;
 
-                if (gf_is_local_addr (dst_brickinfo->hostname)) {
+                if (gf_uuid_compare (dst_brickinfo->uuid, MY_UUID)) {
                         gf_msg_debug (this->name, 0, "I AM THE DESTINATION HOST");
-                        (void) glusterd_brick_disconnect (src_brickinfo);
-                        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo,
-                                                    src_brickinfo, priv);
-                        ret = glusterd_service_stop ("brick", pidfile,
-                                                     SIGTERM, _gf_false);
-                        if (ret == 0) {
-                                glusterd_set_brick_status
-                                        (src_brickinfo, GF_BRICK_STOPPED);
-                                (void) glusterd_brick_unlink_socket_file
-                                                (volinfo, src_brickinfo);
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_BRICK_CLEANUP_SUCCESS,
-                                "Brick cleanup successful.");
-                        } else {
+                        ret = glusterd_volume_stop_glusterfs (volinfo,
+                                                              src_brickinfo,
+                                                              _gf_true);
+                        if (ret) {
                                 gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                        GD_MSG_BRK_CLEANUP_FAIL,
-                                        "Unable to cleanup src brick");
+                                        GD_MSG_BRICK_STOP_FAIL,
+                                        "Unable to stop brick: %s:%s",
+                                        src_brickinfo->hostname,
+                                        src_brickinfo->path);
                                 goto out;
                         }
                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index c75a1011fb3..da015236686 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -2834,8 +2834,6 @@ glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
 
         GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
         if (gf_is_service_running (pidfile, &pid)) {
-                int send_attach_req (xlator_t *this, struct rpc_clnt *rpc,
-                                     char *path, int op);
                 (void) send_attach_req (this, brickinfo->rpc,
                                         brickinfo->path,
                                         GLUSTERD_BRICK_TERMINATE);
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index e7fd01aabb2..01c4f28c7a2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -96,8 +96,8 @@
 int
 send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op);
 
-static gf_boolean_t
-is_brick_mx_enabled ()
+gf_boolean_t
+is_brick_mx_enabled (void)
 {
         char            *value = NULL;
         int             ret = 0;
@@ -1332,7 +1332,8 @@ out:
 int
 glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                                         uuid_t volume_id, char **op_errstr,
-                                        gf_boolean_t is_force)
+                                        gf_boolean_t is_force,
+                                        gf_boolean_t ignore_partition)
 {
         int          ret                 = -1;
         char         parentdir[PATH_MAX] = {0,};
@@ -1406,8 +1407,14 @@ glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                                   " Or use 'force' at the end of the command if"
                                   " you want to override this behavior.",
                                   brickinfo->hostname, brickinfo->path);
-                        ret = -1;
-                        goto out;
+
+                        /* If --wignore-partition flag is used, ignore warnings
+                         * related to bricks being on root partition when 'force'
+                         * is not used */
+                        if (!ignore_partition) {
+                                ret = -1;
+                                goto out;
+                        }
                 }
         }
 
@@ -2077,13 +2084,15 @@ glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
 }
 
 int32_t
-glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo,
+glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo,
+                                glusterd_brickinfo_t *brickinfo,
                                 gf_boolean_t del_brick)
 {
         xlator_t        *this                   = NULL;
-        int             ret                     = 0;
+        glusterd_conf_t *conf                   = NULL;
+        int             ret                     = -1;
         char            *op_errstr              = NULL;
+        char            pidfile[PATH_MAX]       = {0,};
 
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
@@ -2091,6 +2100,11 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
         this = THIS;
         GF_ASSERT (this);
 
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+
+        ret = 0;
+
         if (del_brick)
                 cds_list_del_init (&brickinfo->brick_list);
 
@@ -2105,10 +2119,17 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
                  * an actual signal instead.
                  */
                 if (is_brick_mx_enabled ()) {
+                        gf_msg_debug (this->name, 0, "About to send detach "
+                                      "request for brick %s:%s",
+                                      brickinfo->hostname, brickinfo->path);
+
                         (void) send_attach_req (this, brickinfo->rpc,
                                                 brickinfo->path,
                                                 GLUSTERD_BRICK_TERMINATE);
                 } else {
+                        gf_msg_debug (this->name, 0, "About to stop glusterfsd"
+                                      " for brick %s:%s", brickinfo->hostname,
+                                      brickinfo->path);
                         (void) glusterd_brick_terminate (volinfo, brickinfo,
                                                          NULL, 0, &op_errstr);
                         if (op_errstr) {
@@ -2122,6 +2143,10 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
         if (del_brick)
                 glusterd_delete_brick (volinfo, brickinfo);
 
+        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, conf);
+        gf_msg_debug (this->name,  0, "Unlinking pidfile %s", pidfile);
+        (void) sys_unlink (pidfile);
+out:
         return ret;
 }
 
@@ -6564,9 +6589,6 @@ glusterd_brick_stop (glusterd_volinfo_t *volinfo,
                 goto out;
         }
 
-        gf_msg_debug (this->name, 0, "About to stop glusterfs"
-                " for brick %s:%s", brickinfo->hostname,
-                brickinfo->path);
         ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo, del_brick);
         if (ret) {
                 gf_msg (this->name, GF_LOG_CRITICAL, 0,
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 31bf462bcad..610d17b09a2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -80,6 +80,8 @@ typedef struct glusterd_dict_ctx_ {
         char    *prefix;
 } glusterd_dict_ctx_t;
 
+gf_boolean_t is_brick_mx_enabled (void);
+
 int
 glusterd_compare_lines (const void *a, const void *b);
 
@@ -179,10 +181,13 @@ glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
                                  gf_boolean_t wait);
 
 int32_t
-glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo,
+glusterd_volume_stop_glusterfs (glusterd_volinfo_t *volinfo,
+                                glusterd_brickinfo_t *brickinfo,
                                 gf_boolean_t del_brick);
 
+int
+send_attach_req (xlator_t *this, struct rpc_clnt *rpc, char *path, int op);
+
 glusterd_volinfo_t *
 glusterd_volinfo_ref (glusterd_volinfo_t *volinfo);
 
@@ -310,7 +315,8 @@ glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
 int
 glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
                                         uuid_t volume_id, char **op_errstr,
-                                        gf_boolean_t is_force);
+                                        gf_boolean_t is_force,
+                                        gf_boolean_t ignore_partition);
 int
 glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
                                            int old_state, int new_state,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index a2495602cff..7bd728b7a31 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -1314,7 +1314,7 @@ glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr,
 #endif
                         ret = glusterd_validate_and_create_brickpath (brick_info,
                                                           volume_uuid, op_errstr,
-                                                          is_force);
+                                                          is_force, _gf_false);
                         if (ret)
                                 goto out;