afr: perform replace-brick in a synctask

Backport of http://review.gluster.org/#/c/12169/

Problem:
replace-brick setxattr is not performed inside a synctask. This can lead
to hangs if the setxattr is executed by epoll thread, as the epoll
thread will be waiting for replies to come where as epoll thread is the
thread that needs to epoll_ctl for reading from socket and listen.

Fix:
Move replace-brick to synctask to prevent epoll thread hang.

This patch is in line with the fix performed in
http://review.gluster.org/#/c/12163/

Change-Id: I7284930ead9b0adaa0257f21ec2d893fa5a7146f
BUG: 1262547
Signed-off-by: Ravishankar N <ravishankar@redhat.com>
Reviewed-on: http://review.gluster.org/12172
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>

4 files changed, 73 insertions, 14 deletions

diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 18dd5529fb4..d800a3476d0 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1139,27 +1139,45 @@ out:
 }
 
 int
-_afr_handle_replace_brick (xlator_t *this, call_frame_t *frame, loc_t *loc,
-                           int rb_index)
+_afr_handle_replace_brick_cbk (int ret, call_frame_t *frame, void *opaque)
+{
+        afr_replace_brick_args_t *data = NULL;
+
+        data = opaque;
+        loc_wipe (&data->loc);
+        GF_FREE (data);
+        return 0;
+}
+
+int
+_afr_handle_replace_brick (void *opaque)
 {
 
         afr_local_t     *local          = NULL;
         afr_private_t   *priv           = NULL;
+        int              rb_index       = -1;
         int              ret            = -1;
         int              op_errno       = ENOMEM;
+        call_frame_t    *frame          = NULL;
+        xlator_t        *this           = NULL;
+        afr_replace_brick_args_t *data  = NULL;
 
+        data = opaque;
+        frame = data->frame;
+        rb_index = data->rb_index;
+        this = frame->this;
         priv = this->private;
 
         local = AFR_FRAME_INIT (frame, op_errno);
         if (!local)
                 goto out;
 
-        loc_copy (&local->loc, loc);
+        loc_copy (&local->loc, &data->loc);
 
         gf_log (this->name, GF_LOG_DEBUG, "Child being replaced is : %s",
                 priv->children[rb_index]->name);
 
-        ret = _afr_handle_replace_brick_type (this, frame, loc, rb_index,
+        ret = _afr_handle_replace_brick_type (this, frame, &local->loc, rb_index,
                                               AFR_METADATA_TRANSACTION);
         if (ret) {
                 op_errno = -ret;
@@ -1172,7 +1190,7 @@ _afr_handle_replace_brick (xlator_t *this, call_frame_t *frame, loc_t *loc,
         local->pending = NULL;
         local->xdata_req = NULL;
 
-        ret = _afr_handle_replace_brick_type (this, frame, loc, rb_index,
+        ret = _afr_handle_replace_brick_type (this, frame, &local->loc, rb_index,
                                               AFR_ENTRY_TRANSACTION);
         if (ret) {
                 op_errno = -ret;
@@ -1399,32 +1417,60 @@ afr_handle_replace_brick (xlator_t *this, call_frame_t *frame, loc_t *loc,
 {
         int             ret               = -1;
         int             rb_index          = -1;
+        int             op_errno          = EPERM;
         char           *replace_brick     = NULL;
+        afr_replace_brick_args_t *data    = NULL;
 
         ret =  dict_get_str (dict, GF_AFR_REPLACE_BRICK, &replace_brick);
 
         if (!ret) {
                 if (frame->root->pid != GF_CLIENT_PID_AFR_SELF_HEALD) {
+                        gf_log (this->name, GF_LOG_ERROR, "'%s' is an internal"
+                                " extended attribute : %s.",
+                                GF_AFR_REPLACE_BRICK, strerror (EPERM));
                         ret = 1;
                         goto out;
                 }
                 rb_index = afr_get_child_index_from_name (this, replace_brick);
 
-                if (rb_index < 0)
+                if (rb_index < 0) {
                          /* Didn't belong to this replica pair
                           * Just do a no-op
                           */
                         AFR_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                else
-                        _afr_handle_replace_brick (this, frame, loc, rb_index);
+                        return 0;
+                } else {
+                        data = GF_CALLOC (1, sizeof (*data),
+                                          gf_afr_mt_replace_brick_t);
+                        if (!data) {
+                                ret = 1;
+                                op_errno = ENOMEM;
+                                goto out;
+                        }
+                        data->frame = frame;
+                        loc_copy (&data->loc, loc);
+                        data->rb_index = rb_index;
+                        ret = synctask_new (this->ctx->env,
+                                            _afr_handle_replace_brick,
+                                            _afr_handle_replace_brick_cbk,
+                                            NULL, data);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        AFR_MSG_REPLACE_BRICK_FAILED,
+                                        "Failed to create synctask. Unable to "
+                                        "perform replace-brick.");
+                                ret = 1;
+                                op_errno = ENOMEM;
+                                loc_wipe (&data->loc);
+                                GF_FREE (data);
+                                goto out;
+                        }
+                }
                 ret = 0;
         }
 out:
         if (ret == 1) {
-                gf_log (this->name, GF_LOG_ERROR, "'%s' is an internal"
-                        " extended attribute : %s.",
-                        GF_AFR_REPLACE_BRICK, strerror (EPERM));
-                AFR_STACK_UNWIND (setxattr, frame, -1, EPERM, NULL);
+                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
                 ret = 0;
         }
         return ret;
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index fd484e4f1fd..6f1eee95322 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -45,6 +45,7 @@ enum gf_afr_mem_types_ {
 	gf_afr_mt_subvol_healer_t,
 	gf_afr_mt_spbc_timeout_t,
         gf_afr_mt_spb_status_t,
+        gf_afr_mt_replace_brick_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
index 4793413ed83..d07c7bc2170 100644
--- a/xlators/cluster/afr/src/afr-messages.h
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -45,7 +45,7 @@
  */
 
 #define GLFS_COMP_BASE_AFR      GLFS_MSGID_COMP_AFR
-#define GLFS_NUM_MESSAGES       38
+#define GLFS_NUM_MESSAGES       39
 #define GLFS_MSGID_END          (GLFS_COMP_BASE_AFR + GLFS_NUM_MESSAGES + 1)
 
 #define glfs_msg_start_x GLFS_COMP_BASE_AFR, "Invalid: Start of messages"
@@ -332,7 +332,6 @@
 */
 #define AFR_MSG_SELF_HEAL_FAILED                (GLFS_COMP_BASE_AFR + 37)
 
-
 /*!
  * @messageid 108038
  * @diagnosis
@@ -340,6 +339,13 @@
 */
 #define AFR_MSG_SPLIT_BRAIN_STATUS      (GLFS_COMP_BASE_AFR + 38)
 
+/*!
+ * @messageid 108039
+ * @diagnosis
+ * @recommendedaction
+*/
+#define AFR_MSG_REPLACE_BRICK_FAILED                (GLFS_COMP_BASE_AFR + 39)
+
 
 #define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 #endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 870d26b7689..6a14febec0a 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -760,6 +760,12 @@ typedef struct afr_spb_status {
         loc_t        *loc;
 } afr_spb_status_t;
 
+typedef struct afr_replace_brick_args {
+        call_frame_t *frame;
+        loc_t loc;
+        int rb_index;
+} afr_replace_brick_args_t;
+
 typedef struct afr_read_subvol_args {
         ia_type_t ia_type;
         uuid_t gfid;