glusterd / afr : Enable auto heal when replica count increases

        Backport of http://review.gluster.org/12451

In replicate volumes, when a brick is added to a replicate
group, heal to the new brick should be triggered.
Also, the new brick should not be considered as source for
healing till it is up to date.

Previously, extended attributes had to be set manually on
the bricks for this to happen. This patch is part 1 patch
to automate this process.

        >Change-Id: I29958448618372bfde23bf1dac5dd23dba1ad98f
        >BUG: 1276203
        >Signed-off-by: Anuradha Talur <atalur@redhat.com>
        >Reviewed-on: http://review.gluster.org/12451
        >Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
        >NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
        >CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
        >Reviewed-by: Ravishankar N <ravishankar@redhat.com>
        >Smoke: Gluster Build System <jenkins@build.gluster.com>
Signed-off-by: Anuradha Talur <atalur@redhat.com>

Conflicts:
	libglusterfs/src/globals.h
	xlators/mgmt/glusterd/src/glusterd-replace-brick.c

Change-Id: Ica83592aab8edbe49e2bb9d8d4824cf5c76324b7
BUG: 1320020
Reviewed-on: http://review.gluster.org/13806
Smoke: Gluster Build System <jenkins@build.gluster.com>
Tested-by: Anuradha Talur <atalur@redhat.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

8 files changed, 280 insertions, 91 deletions

diff --git a/libglusterfs/src/globals.h b/libglusterfs/src/globals.h
index 9d2ca8d2241..364925d7f8f 100644
--- a/libglusterfs/src/globals.h
+++ b/libglusterfs/src/globals.h
@@ -37,7 +37,7 @@
  */
 #define GD_OP_VERSION_MIN  1 /* MIN is the fresh start op-version, mostly
                                 should not change */
-#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_7_7 /* MAX VERSION is the maximum
+#define GD_OP_VERSION_MAX  GD_OP_VERSION_3_7_10 /* MAX VERSION is the maximum
                                                   count in VME table, should
                                                   keep changing with
                                                   introduction of newer
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 81caf12e62b..d1db087c3ca 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -176,6 +176,7 @@
 #define GF_AFR_SBRAIN_CHOICE "replica.split-brain-choice"
 #define GF_AFR_SPB_CHOICE_TIMEOUT "replica.split-brain-choice-timeout"
 #define GF_AFR_SBRAIN_RESOLVE "replica.split-brain-heal-finalize"
+#define GF_AFR_ADD_BRICK "trusted.add-brick"
 #define GF_AFR_REPLACE_BRICK "trusted.replace-brick"
 #define GF_AFR_DIRTY "trusted.afr.dirty"
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 99b5261e9c1..aaa6a1a8895 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -26,6 +26,7 @@
 #include "glusterd-messages.h"
 #include "glusterd-server-quorum.h"
 #include "run.h"
+#include "glusterd-volgen.h"
 #include <sys/signal.h>
 
 /* misc */
@@ -1238,6 +1239,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         char                         *brick_mount_dir  = NULL;
         xlator_t                     *this           = NULL;
         glusterd_conf_t              *conf           = NULL;
+        gf_boolean_t                  is_valid_add_brick = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1325,6 +1327,7 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         /* Gets changed only if the options are given in add-brick cli */
         if (type)
                 volinfo->type = type;
+
         if (replica_count) {
                 volinfo->replica_count = replica_count;
         }
@@ -1360,6 +1363,27 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                         CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
 #endif
 
+        /* This check needs to be added to distinguish between
+         * attach-tier commands and add-brick commands.
+         * When a tier is attached, adding is done via add-brick
+         * and setting of pending xattrs shouldn't be done for
+         * attach-tiers as they are virtually new volumes.
+         */
+        if (glusterd_is_volume_replicate (volinfo)) {
+                if (replica_count &&
+                    !dict_get (dict, "attach-tier") &&
+                    conf->op_version >= GD_OP_VERSION_3_7_10) {
+                        is_valid_add_brick = _gf_true;
+                        ret = generate_dummy_client_volfiles (volinfo);
+                        if (ret) {
+                                gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_VOLFILE_CREATE_FAIL,
+                                        "Failed to create volfile.");
+                                goto out;
+                                }
+                        }
+        }
+
         while (i <= count) {
                 ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
                                                               &brickinfo);
@@ -1391,6 +1415,16 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                         }
                 }
 
+                /* if the volume is a replicate volume, do: */
+                if (is_valid_add_brick) {
+                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
+                                ret = glusterd_handle_replicate_brick_ops (
+                                                           volinfo, brickinfo,
+                                                           GD_OP_ADD_BRICK);
+                                if (ret < 0)
+                                        goto out;
+                        }
+                }
                 ret = glusterd_brick_start (volinfo, brickinfo,
                                             _gf_true);
                 if (ret)
@@ -1519,22 +1553,6 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         conf = this->private;
         GF_ASSERT (conf);
 
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (ret) {
-                gf_msg_debug (THIS->name, 0,
-                        "Unable to get replica count");
-        }
-
-        if (replica_count > 0) {
-                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                        msg, sizeof(msg));
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_msg (THIS->name, GF_LOG_ERROR, errno,
@@ -1555,6 +1573,42 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+        if (ret) {
+                gf_msg_debug (THIS->name, 0,
+                        "Unable to get replica count");
+        }
+
+        if (replica_count > 0) {
+                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
+                                        msg, sizeof(msg));
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_OP_VERSION_MISMATCH, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+        }
+
+        /* Do not allow add-brick for stopped volumes when replica-count
+         * is being increased.
+         */
+        if (glusterd_is_volume_replicate (volinfo)) {
+                if (conf->op_version >= GD_OP_VERSION_3_7_10 &&
+                    !dict_get (dict, "attach-tier") &&
+                    replica_count &&
+                    GLUSTERD_STATUS_STOPPED == volinfo->status) {
+                        ret = -1;
+                        snprintf (msg, sizeof (msg), " Volume must not be in"
+                                  " stopped state when replica-count needs to "
+                                  " be increased.");
+                        gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                                GD_MSG_BRICK_ADD_FAIL, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        goto out;
+                }
+        }
+
         if (conf->op_version > GD_OP_VERSION_3_7_5 &&
             is_origin_glusterd (dict)) {
                 ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index a3620bd33b3..3b8483847e4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -44,78 +44,6 @@ glusterd_mgmt_v3_initiate_replace_brick_cmd_phases (rpcsvc_request_t *req,
                                                     glusterd_op_t op,
                                                     dict_t *dict);
 int
-glusterd_handle_replicate_replace_brick (glusterd_volinfo_t *volinfo,
-                                         glusterd_brickinfo_t *brickinfo)
-{
-        int32_t                    ret               = -1;
-        char                       tmpmount[]        = "/tmp/mntXXXXXX";
-        char                       logfile[PATH_MAX] = {0,};
-        int                        dirty[3]          = {0,};
-        runner_t                   runner            = {0};
-        glusterd_conf_t           *priv              = NULL;
-        char                      *pid               = NULL;
-
-        priv = THIS->private;
-
-        dirty[2] = hton32(1);
-
-        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,
-                             sizeof (dirty), 0);
-        if (ret == -1) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
-                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));
-                goto out;
-        }
-
-        if (mkdtemp (tmpmount) == NULL) {
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_DIR_OP_FAILED,
-                        "failed to create a temporary mount directory.");
-                ret = -1;
-                goto out;
-        }
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",
-                  volinfo->volname);
-
-        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);
-        if (ret < 0)
-                goto out;
-
-        runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", "localhost",
-                         "--volfile-id", volinfo->volname,
-                         "--client-pid", pid,
-                         "-l", logfile, tmpmount, NULL);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-
-        if (ret) {
-                runner_log (&runner, THIS->name, GF_LOG_ERROR, "mount command"
-                            "failed.");
-                goto lock;
-        }
-        ret = sys_lsetxattr (tmpmount, GF_AFR_REPLACE_BRICK,
-                             brickinfo->brick_id, sizeof (brickinfo->brick_id),
-                             0);
-        if (ret == -1)
-                gf_msg (THIS->name, GF_LOG_ERROR, errno,
-                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
-                        " attribute %s : %s", GF_AFR_REPLACE_BRICK,
-                        strerror (errno));
-        gf_umount_lazy (THIS->name, tmpmount, 1);
-lock:
-        synclock_lock (&priv->big_lock);
-out:
-        if (pid)
-                GF_FREE (pid);
-        gf_msg_debug ("glusterd", 0, "Returning with ret");
-        return ret;
-}
-
-int
 __glusterd_handle_replace_brick (rpcsvc_request_t *req)
 {
         int32_t                         ret = -1;
@@ -670,8 +598,8 @@ glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
         /* if the volume is a replicate volume, do: */
         if (glusterd_is_volume_replicate (volinfo)) {
                 if (!gf_uuid_compare (new_brickinfo->uuid, MY_UUID)) {
-                        ret = glusterd_handle_replicate_replace_brick
-                                  (volinfo, new_brickinfo);
+                        ret = glusterd_handle_replicate_brick_ops (volinfo,
+                                        new_brickinfo, GD_OP_REPLACE_BRICK);
                         if (ret < 0)
                                 goto out;
                 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index cfee26dfe7e..c114f0349c7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7097,6 +7097,33 @@ void glusterd_update_tier_status (glusterd_volinfo_t *volinfo) {
 }
 
 int
+glusterd_get_dummy_client_filepath (char *filepath,
+                                    glusterd_volinfo_t *volinfo,
+                                    gf_transport_type type)
+{
+        int   ret             = 0;
+        char  path[PATH_MAX]  = {0,};
+
+        switch (type) {
+        case GF_TRANSPORT_TCP:
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+                snprintf (filepath, PATH_MAX,
+                          "/tmp/%s.tcp-fuse.vol", volinfo->volname);
+                break;
+
+        case GF_TRANSPORT_RDMA:
+                snprintf (filepath, PATH_MAX,
+                          "/tmp/%s.rdma-fuse.vol", volinfo->volname);
+                break;
+        default:
+                ret = -1;
+                break;
+        }
+
+        return ret;
+}
+
+int
 glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
                               size_t len, int cmd, defrag_cbk_fn_t cbk)
 {
@@ -11052,3 +11079,117 @@ gd_get_shd_key (int type)
         }
         return key;
 }
+
+int
+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,
+                                     glusterd_brickinfo_t *brickinfo,
+                                     glusterd_op_t op)
+{
+        int32_t                    ret               = -1;
+        char                       tmpmount[]        = "/tmp/mntXXXXXX";
+        char                       logfile[PATH_MAX] = {0,};
+        int                        dirty[3]          = {0,};
+        runner_t                   runner            = {0};
+        glusterd_conf_t           *priv              = NULL;
+        char                      *pid               = NULL;
+        char                       vpath[PATH_MAX]   = {0,};
+        char                      *volfileserver     = NULL;
+
+        priv = THIS->private;
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+
+        dirty[2] = hton32(1);
+
+        ret = sys_lsetxattr (brickinfo->path, GF_AFR_DIRTY, dirty,
+                             sizeof (dirty), 0);
+        if (ret == -1) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
+                        " attribute %s : %s.", GF_AFR_DIRTY, strerror (errno));
+                goto out;
+        }
+
+        if (mkdtemp (tmpmount) == NULL) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DIR_OP_FAILED,
+                        "failed to create a temporary mount directory.");
+                ret = -1;
+                goto out;
+        }
+
+        ret = gf_asprintf (&pid, "%d", GF_CLIENT_PID_SELF_HEALD);
+        if (ret < 0)
+                goto out;
+
+        switch (op) {
+        case GD_OP_REPLACE_BRICK:
+        if (dict_get_str (THIS->options, "transport.socket.bind-address",
+                          &volfileserver) != 0)
+                volfileserver = "localhost";
+
+                snprintf (logfile, sizeof (logfile),
+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-replace-brick-mount.log",
+                          volinfo->volname);
+                if (!*logfile) {
+                        ret = -1;
+                        goto out;
+                }
+                runinit (&runner);
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "-s", volfileserver,
+                                 "--volfile-id", volinfo->volname,
+                                 "--client-pid", pid,
+                                 "-l", logfile, tmpmount, NULL);
+                break;
+
+        case GD_OP_ADD_BRICK:
+                snprintf (logfile, sizeof (logfile),
+                          DEFAULT_LOG_FILE_DIRECTORY"/%s-add-brick-mount.log",
+                          volinfo->volname);
+                if (!*logfile) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_get_dummy_client_filepath (vpath, volinfo,
+                                                    volinfo->transport_type);
+                if (ret) {
+                        gf_log ("", GF_LOG_ERROR, "Failed to get "
+                                "volfile path");
+                        goto out;
+                }
+                runinit (&runner);
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "--volfile", vpath,
+                                 "--client-pid", pid,
+                                 "-l", logfile, tmpmount, NULL);
+                break;
+        default:
+                break;
+        }
+        synclock_unlock (&priv->big_lock);
+        ret = runner_run (&runner);
+
+        if (ret) {
+                gf_log (THIS->name, GF_LOG_ERROR, "mount command"
+                        " failed.");
+                goto lock;
+        }
+        ret = sys_lsetxattr (tmpmount, (op == GD_OP_REPLACE_BRICK) ?
+                             GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,
+                             brickinfo->brick_id, sizeof (brickinfo->brick_id),
+                             0);
+        if (ret == -1)
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SETXATTR_FAIL, "Failed to set extended"
+                        " attribute %s : %s", (op == GD_OP_REPLACE_BRICK) ?
+                        GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,
+                        strerror (errno));
+        gf_umount_lazy (THIS->name, tmpmount, 1);
+lock:
+        synclock_lock (&priv->big_lock);
+out:
+        if (pid)
+                GF_FREE (pid);
+        gf_msg_debug ("glusterd", 0, "Returning with ret");
+        return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index e6380f596e0..6b74e900c17 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -695,4 +695,14 @@ int
 glusterd_volume_brick_for_each (glusterd_volinfo_t *volinfo, void *data,
                int (*fn) (glusterd_volinfo_t *, glusterd_brickinfo_t *,
                           dict_t *mod_dict, void *));
+
+int
+glusterd_get_dummy_client_filepath (char *filepath,
+                                    glusterd_volinfo_t *volinfo,
+                                    gf_transport_type type);
+
+int
+glusterd_handle_replicate_brick_ops (glusterd_volinfo_t *volinfo,
+                                     glusterd_brickinfo_t *brickinfo,
+                                     glusterd_op_t op);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index f1d9217be59..4db3538162f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -5205,6 +5205,58 @@ enumerate_transport_reqs (gf_transport_type type, char **types)
 }
 
 int
+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo)
+{
+        int                i                  = 0;
+        int                ret                = -1;
+        char               filepath[PATH_MAX] = {0,};
+        char               *types[]           = {NULL, NULL, NULL};
+        dict_t             *dict              = NULL;
+        xlator_t           *this              = NULL;
+        gf_transport_type  type               = GF_TRANSPORT_TCP;
+
+        this = THIS;
+
+        enumerate_transport_reqs (volinfo->transport_type, types);
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+        for (i = 0; types[i]; i++) {
+                memset (filepath, 0, sizeof (filepath));
+                ret = dict_set_str (dict, "client-transport-type", types[i]);
+                if (ret)
+                        goto out;
+                type = transport_str_to_type (types[i]);
+
+                ret = dict_set_uint32 (dict, "trusted-client", GF_CLIENT_OTHER);
+                if (ret)
+                        goto out;
+
+                ret = glusterd_get_dummy_client_filepath (filepath,
+                                                          volinfo, type);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, EINVAL,
+                                GD_MSG_INVALID_ENTRY,
+                                "Received invalid transport-type.");
+                        goto out;
+                }
+
+                ret = generate_single_transport_client_volfile (volinfo,
+                                                                filepath,
+                                                                dict);
+                if (ret)
+                        goto out;
+        }
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        gf_msg_trace ("glusterd", 0, "Returning %d", ret);
+        return ret;
+}
+
+int
 generate_client_volfiles (glusterd_volinfo_t *volinfo,
                           glusterd_client_type_t client_type)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index f1dc823c719..c86a87b8eb1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -293,4 +293,7 @@ glusterd_volopt_validate (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
 gf_boolean_t
 gd_is_self_heal_enabled (glusterd_volinfo_t *volinfo, dict_t *dict);
 
+int
+generate_dummy_client_volfiles (glusterd_volinfo_t *volinfo);
+
 #endif