glusterd/add-brick: change add-brick implementation to v3 framework

add-brick commit first happens on local node and followed by peers.
As part of the commit of local-host glusterd will send the updated
volfiles to the clients connected to the local-host even before the
commit of peers happen. If any of the newly added brick was hosted
by any peer, that brick won't be started when client (connected to
local-host) try to send fops.

By changing to v3 framework we can send post validate ops
after commit operation that helps to send volfile fetch request only
after completing commits on all nodes.

back port of:
>Change-Id: Ib7312e01143326128c010c11fc2ed206f37409ad
>BUG: 1263549
>Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
>Reviewed-on: http://review.gluster.org/12237
>Tested-by: NetBSD Build System <jenkins@build.gluster.org>
>Tested-by: Gluster Build System <jenkins@build.gluster.com>
>Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

(cherry picked from commit ee944e86866a6556fd4dd98bcd6f1f58c323721f)

Change-Id: Idfa993f2c94a52c2a30be525eeac66af1c320059
BUG: 1259081
Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com>
Reviewed-on: http://review.gluster.org/12308
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>

2 files changed, 134 insertions, 17 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 282a78a0e49..cb206ba9db7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -19,10 +19,12 @@
 #include "glusterd-op-sm.h"
 #include "glusterd-geo-rep.h"
 #include "glusterd-store.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-svc-helper.h"
 #include "glusterd-messages.h"
+#include "glusterd-server-quorum.h"
 #include "run.h"
 #include <sys/signal.h>
 
@@ -417,11 +419,16 @@ __glusterd_handle_add_brick (rpcsvc_request_t *req)
         int32_t                         replica_count = 0;
         int32_t                         stripe_count = 0;
         int                             type = 0;
+        glusterd_conf_t                 *conf = NULL;
+
         this = THIS;
         GF_ASSERT(this);
 
         GF_ASSERT (req);
 
+        conf = this->private;
+        GF_ASSERT (conf);
+
         ret = xdr_to_generic (req->msg[0], &cli_req,
                               (xdrproc_t)xdr_gf_cli_req);
         if (ret < 0) {
@@ -640,7 +647,17 @@ brick_val:
                 }
         }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+                gf_msg_debug (this->name, 0, "The cluster is operating at "
+                          "version less than or equal to %d. Falling back "
+                          "to syncop framework.",
+                          GD_OP_VERSION_3_7_5);
+                ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+        } else {
+                ret = glusterd_mgmt_v3_initiate_all_phases (req,
+                                                            GD_OP_ADD_BRICK,
+                                                            dict);
+        }
 
 out:
         if (ret) {
@@ -1323,14 +1340,14 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
         volinfo->subvol_count = (volinfo->brick_count /
                                  volinfo->dist_leaf_count);
 
-        ret = glusterd_create_volfiles (volinfo);
-        if (ret)
-                goto out;
-
         ret = 0;
         if (GLUSTERD_STATUS_STARTED != volinfo->status)
                 goto out;
 
+        ret = generate_brick_volfiles (volinfo);
+        if (ret)
+                goto out;
+
         brick_list = gf_strdup (bricks);
         free_ptr2 = brick_list;
         i = 1;
@@ -1401,8 +1418,16 @@ glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
                               _glusterd_restart_gsync_session, &param);
         }
         volinfo->caps = caps;
-
-        ret = glusterd_fetchspec_notify (this);
+        if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+               ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        } else {
+                /*
+                 * The cluster is operating at version greater than
+                 * gluster-3.7.5. So no need to sent volfile fetch
+                 * request in commit phase, the same will be done
+                 * in post validate phase with v3 framework.
+                 */
+        }
 
 out:
         GF_FREE (free_ptr1);
@@ -1528,6 +1553,28 @@ glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         if (ret)
                 goto out;
 
+        if (conf->op_version > GD_OP_VERSION_3_7_5 &&
+            is_origin_glusterd (dict)) {
+                ret = glusterd_validate_quorum (this, GD_OP_ADD_BRICK, dict,
+                                                op_errstr);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
+                                GD_MSG_SERVER_QUORUM_NOT_MET,
+                                "Server quorum not met. Rejecting operation.");
+                        goto out;
+                }
+        } else {
+                /* Case 1: conf->op_version <= GD_OP_VERSION_3_7_5
+                 *         in this case the add-brick is running
+                 *         syncop framework that will do a quorum
+                 *         check by default
+                 * Case 2: We don't need to do quorum check on every
+                 *         node, only originator glusterd need to
+                 *         check for quorum
+                 * So nothing need to be done in else
+                 */
+        }
+
         if (glusterd_is_defrag_on(volinfo)) {
                 snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
                           "progress. Please retry after completion", volname);
@@ -2216,10 +2263,19 @@ glusterd_op_add_brick (dict_t *dict, char **op_errstr)
                         GD_MSG_BRICK_ADD_FAIL, "Unable to add bricks");
                 goto out;
         }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
+        if (priv->op_version <= GD_OP_VERSION_3_7_5) {
+               ret = glusterd_store_volinfo (volinfo,
+                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                if (ret)
+                        goto out;
+        } else {
+                 /*
+                 * The cluster is operating at version greater than
+                 * gluster-3.7.5. So no need to store volfiles
+                 * in commit phase, the same will be done
+                 * in post validate phase with v3 framework.
+                 */
+        }
 
         if (GLUSTERD_STATUS_STARTED == volinfo->status)
                 ret = glusterd_svcs_manager (volinfo);
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index 326d8681495..f65c3a39f99 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -179,6 +179,15 @@ gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
                         goto out;
                 }
                 break;
+        case GD_OP_ADD_BRICK:
+                ret = glusterd_op_stage_add_brick (dict, op_errstr, rsp_dict);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_WARNING, 0,
+                                GD_MSG_PRE_VALIDATION_FAIL,
+                                "Replace-brick prevalidation failed.");
+                        goto out;
+                }
+                break;
 
         default:
                 break;
@@ -264,6 +273,18 @@ gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
                         }
                         break;
                 }
+                case GD_OP_ADD_BRICK:
+                {
+                        ret = glusterd_op_add_brick (dict, op_errstr);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0,
+                                        GD_MSG_COMMIT_OP_FAIL,
+                                        "Add-brick commit failed.");
+                                goto out;
+                        }
+                        break;
+
+                }
                default:
                        break;
         }
@@ -278,8 +299,11 @@ int32_t
 gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
                              char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
+        int32_t                  ret       = -1;
+        xlator_t                *this      = NULL;
+        char                    *volname   = NULL;
+        glusterd_volinfo_t      *volinfo   = NULL;
+
 
         this = THIS;
         GF_ASSERT (this);
@@ -301,8 +325,36 @@ gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
                        }
                        break;
                }
+               case GD_OP_ADD_BRICK:
+               {
+                        ret = dict_get_str (dict, "volname", &volname);
+                        if (ret) {
+                                gf_msg ("glusterd", GF_LOG_ERROR, errno,
+                                        GD_MSG_DICT_GET_FAILED, "Unable to get"
+                                        " volume name");
+                                goto out;
+                        }
+
+                        ret = glusterd_volinfo_find (volname, &volinfo);
+                        if (ret) {
+                                gf_msg ("glusterd", GF_LOG_ERROR, EINVAL,
+                                        GD_MSG_VOL_NOT_FOUND, "Unable to "
+                                        "allocate memory");
+                                goto out;
+                        }
+                        ret = glusterd_create_volfiles_and_notify_services (
+                                                                     volinfo);
+                        if (ret)
+                                goto out;
+                        ret = glusterd_store_volinfo (volinfo,
+                                            GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+                        if (ret)
+                                goto out;
+                        break;
+
+               }
                default:
-                       break;
+                        break;
         }
 
         ret = 0;
@@ -559,6 +611,15 @@ glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op,
                         goto out;
                 }
                 break;
+        case GD_OP_ADD_BRICK:
+                ret = glusterd_aggr_brick_mount_dirs (aggr, rsp);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL, "Failed to "
+                                "aggregate brick mount dirs");
+                        goto out;
+                }
+                break;
         default:
                 ret = -1;
                 gf_msg (this->name, GF_LOG_ERROR, EINVAL,
@@ -857,6 +918,7 @@ glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
                 case GD_OP_SNAP:
                         dict_copy (dict, req_dict);
                         break;
+                case GD_OP_ADD_BRICK:
                 case GD_OP_REPLACE_BRICK:
                 {
                         ret = dict_get_str (dict, "volname", &volname);
@@ -1880,9 +1942,8 @@ glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
         }
 
         /* POST-COMMIT VALIDATE PHASE */
-        /* As of now, post_validate is not handling any other
-           commands other than snapshot. So as of now, I am
-           sending 0 (op_ret as 0).
+        /* As of now, post_validate is not trying to cleanup any failed
+           commands. So as of now, I am sending 0 (op_ret as 0).
         */
         ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict, &op_errstr,
                                               txn_generation);