From d7cc779da48e484d0edb7bfbd903f941d0ccca32 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Mon, 3 Sep 2012 15:58:26 +0530 Subject: glusterd: op-version handshake implementation Brings in a new rpc program MGMT_HANDSHAKE, which implements the op-version handshake. This is required for bringing in the op-version feature as described in http://www.gluster.org/community/documentation/index.php/Features/Opversion Change-Id: I4333fd2714dbbd3a2a3fca5862cbb3c56615529e BUG: 814534 Signed-off-by: Kaushal M Reviewed-on: http://review.gluster.org/3688 Reviewed-by: Vijay Bellur Tested-by: Gluster Build System --- xlators/mgmt/glusterd/src/glusterd-handler.c | 2 +- xlators/mgmt/glusterd/src/glusterd-handshake.c | 574 +++++++++++++++++++++++-- xlators/mgmt/glusterd/src/glusterd-store.c | 162 +++++-- xlators/mgmt/glusterd/src/glusterd-store.h | 10 +- xlators/mgmt/glusterd/src/glusterd.c | 71 ++- xlators/mgmt/glusterd/src/glusterd.h | 29 +- 6 files changed, 753 insertions(+), 95 deletions(-) (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index a3106342d..7cdad10e2 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -2881,7 +2881,7 @@ glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata, gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT"); peerinfo->connected = 1; - ret = glusterd_peer_handshake (this, rpc, peerctx); + ret = glusterd_peer_dump_version (this, rpc, peerctx); if (ret) gf_log ("", GF_LOG_ERROR, "glusterd handshake failed"); break; diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c index ab38de17c..428051d4a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handshake.c +++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c @@ -21,6 +21,7 @@ #include "glusterd.h" #include "glusterd-utils.h" #include "glusterd-op-sm.h" +#include "glusterd-store.h" #include "glusterfs3.h" #include "protocol-common.h" @@ -267,34 +268,247 @@ fail: return 0; } + +int +gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version, + char *peerid) +{ + int ret = -1; + glusterd_conf_t *conf = NULL; + + conf = this->private; + + if (cluster_op_version > GD_OP_VERSION_MAX) { + gf_log (this->name, GF_LOG_ERROR, + "operating version %d is more than the maximum " + "supported (%d) on the machine (as per peer request " + "from %s)", cluster_op_version, GD_OP_VERSION_MAX, + peerid); + goto out; + } + + if (cluster_op_version < conf->op_version) { + gf_log (this->name, GF_LOG_ERROR, + "operating version %d is less than the currently " + "running version (%d) on the machine (as per peer " + "request from %s)", cluster_op_version, + conf->op_version, peerid); + goto out; + } + + ret = 0; +out: + return ret; +} + +int +glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req) +{ + dict_t *dict = NULL; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + int ret = -1; + int op_errno = EINVAL; + gf_mgmt_hndsk_req args = {{0,},}; + gf_mgmt_hndsk_rsp rsp = {0,}; + + this = THIS; + conf = this->private; + + if (!xdr_to_generic (req->msg[0], &args, + (xdrproc_t)xdr_gf_mgmt_hndsk_req)) { + //failed to decode msg; + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + dict = dict_new (); + if (!dict) + goto out; + + ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version); + if (ret) { + gf_log (this->name, GF_LOG_WARNING, + "failed to set operating version"); + rsp.op_ret = ret; + goto out; + } + + ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN); + if (ret) { + gf_log (this->name, GF_LOG_WARNING, + "failed to set %s", GD_MIN_OP_VERSION_KEY); + rsp.op_ret = ret; + goto out; + } + + ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX); + if (ret) { + gf_log (this->name, GF_LOG_WARNING, + "failed to set %s", GD_MAX_OP_VERSION_KEY); + rsp.op_ret = ret; + goto out; + } + + ret = 0; + + GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val), + rsp.hndsk.hndsk_len, op_errno, out); +out: + + rsp.op_ret = ret; + rsp.op_errno = op_errno; + + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (xdrproc_t)xdr_gf_mgmt_hndsk_rsp); + + ret = 0; + + if (dict) + dict_unref (dict); + + if (args.hndsk.hndsk_val) + free (args.hndsk.hndsk_val); + + if (rsp.hndsk.hndsk_val) + GF_FREE (rsp.hndsk.hndsk_val); + + return ret; +} + +int +glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req) +{ + dict_t *clnt_dict = NULL; + xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; + int ret = -1; + int op_errno = EINVAL; + int32_t peer_op_version = 0; + gf_mgmt_hndsk_req args = {{0,},}; + gf_mgmt_hndsk_rsp rsp = {0,}; + + this = THIS; + conf = this->private; + + if (!xdr_to_generic (req->msg[0], &args, + (xdrproc_t)xdr_gf_mgmt_hndsk_req)) { + //failed to decode msg; + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val, + (args.hndsk.hndsk_len), ret, op_errno, + out); + + ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, + &peer_op_version); + if (ret) { + gf_log (this->name, GF_LOG_WARNING, + "failed to get the op-version key peer=%s", + req->trans->peerinfo.identifier); + goto out; + } + + ret = gd_validate_cluster_op_version (this, peer_op_version, + req->trans->peerinfo.identifier); + if (ret) + goto out; + + + /* As this is ACK from the Cluster for the versions supported, + can set the op-version of 'this' glusterd to the one + received. */ + gf_log (this->name, GF_LOG_INFO, "using the op-version %d", + peer_op_version); + conf->op_version = peer_op_version; + ret = glusterd_store_global_info (this); + if (ret) + gf_log (this->name, GF_LOG_ERROR, "Failed to store op-version"); + +out: + rsp.op_ret = ret; + rsp.op_errno = op_errno; + + glusterd_submit_reply (req, &rsp, NULL, 0, NULL, + (xdrproc_t)xdr_gf_mgmt_hndsk_rsp); + + if (clnt_dict) + dict_unref (clnt_dict); + + if (args.hndsk.hndsk_val) + free (args.hndsk.hndsk_val); + + return ret; +} + + rpcsvc_actor_t gluster_handshake_actors[] = { - [GF_HNDSK_NULL] = {"NULL", GF_HNDSK_NULL, NULL, NULL, 0}, - [GF_HNDSK_GETSPEC] = {"GETSPEC", GF_HNDSK_GETSPEC, server_getspec, NULL, 0}, - [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, - NULL, 0}, + [GF_HNDSK_NULL] = {"NULL", GF_HNDSK_NULL, NULL, NULL, 0}, + [GF_HNDSK_GETSPEC] = {"GETSPEC", GF_HNDSK_GETSPEC, + server_getspec, NULL, 0}, + [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, + server_event_notify, NULL, 0}, }; struct rpcsvc_program gluster_handshake_prog = { - .progname = "GlusterFS Handshake", + .progname = "Gluster Handshake", .prognum = GLUSTER_HNDSK_PROGRAM, .progver = GLUSTER_HNDSK_VERSION, .actors = gluster_handshake_actors, .numactors = GF_HNDSK_MAXVALUE, }; + char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = { [GF_DUMP_NULL] = "NULL", [GF_DUMP_DUMP] = "DUMP", }; rpc_clnt_prog_t glusterd_dump_prog = { - .progname = "GLUSTERD-DUMP", - .prognum = GLUSTER_DUMP_PROGRAM, - .progver = GLUSTER_DUMP_VERSION, - .procnames = glusterd_dump_proc, + .progname = "GLUSTERD-DUMP", + .prognum = GLUSTER_DUMP_PROGRAM, + .progver = GLUSTER_DUMP_VERSION, + .procnames = glusterd_dump_proc, }; + +rpcsvc_actor_t glusterd_mgmt_hndsk_actors[] = { + [GD_MGMT_HNDSK_NULL] = {"NULL", GD_MGMT_HNDSK_NULL, NULL, + NULL, 0}, + [GD_MGMT_HNDSK_VERSIONS] = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS, + glusterd_mgmt_hndsk_versions, NULL, + 0}, + [GD_MGMT_HNDSK_VERSIONS_ACK] = {"MGMT-VERS-ACK", + GD_MGMT_HNDSK_VERSIONS_ACK, + glusterd_mgmt_hndsk_versions_ack, + NULL, 0}, +}; + +struct rpcsvc_program glusterd_mgmt_hndsk_prog = { + .progname = "Gluster MGMT Handshake", + .prognum = GD_MGMT_HNDSK_PROGRAM, + .progver = GD_MGMT_HNDSK_VERSION, + .actors = glusterd_mgmt_hndsk_actors, + .numactors = GD_MGMT_HNDSK_MAXVALUE, +}; + +char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = { + [GD_MGMT_HNDSK_NULL] = "NULL", + [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS", + [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK", +}; + +rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = { + .progname = "Gluster MGMT Handshake", + .prognum = GD_MGMT_HNDSK_PROGRAM, + .progver = GD_MGMT_HNDSK_VERSION, + .procnames = glusterd_mgmt_hndsk_proc, +}; + + static int glusterd_event_connected_inject (glusterd_peerctx_t *peerctx) { @@ -343,12 +557,279 @@ out: return ret; } + +int +gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo, + dict_t *dict, char **errstr) +{ + int ret = -1; + glusterd_conf_t *conf = NULL; + int32_t peer_op_version = 0; + int32_t peer_min_op_version = 0; + int32_t peer_max_op_version = 0; + + if (!dict && !this && !peerinfo) + goto out; + + conf = this->private; + + ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version); + if (ret) + goto out; + + ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY, + &peer_max_op_version); + if (ret) + goto out; + + ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY, + &peer_min_op_version); + if (ret) + goto out; + + ret = -1; + /* Check if peer can support our op_version */ + if ((peer_max_op_version < conf->op_version) || + (peer_min_op_version > conf->op_version)) { + ret = gf_asprintf (errstr, "Peer %s does not support required " + "op-version", peerinfo->hostname); + ret = -1; + goto out; + } + + /* If peer is already operating at a higher op_version reject it. + * Cluster cannot be moved to higher op_version to accomodate a peer. + */ + if (peer_op_version > conf->op_version) { + ret = gf_asprintf (errstr, "Peer %s is already at a higher " + "op-version", peerinfo->hostname); + ret = -1; + goto out; + } + + ret = 0; +out: + gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname, + ((ret < 0) ? "rejected" : "accepted")); + return ret; +} + +int +glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov, + int count, void *myframe) +{ + int ret = -1; + int op_errno = EINVAL; + gf_mgmt_hndsk_rsp rsp = {0,}; + xlator_t *this = NULL; + call_frame_t *frame = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + glusterd_peerctx_t *peerctx = NULL; + char msg[1024] = {0,}; + + this = THIS; + frame = myframe; + peerctx = frame->local; + peerinfo = peerctx->peerinfo; + + if (-1 == req->rpc_status) { + snprintf (msg, sizeof (msg), + "Error through RPC layer, retry again later"); + gf_log ("", GF_LOG_ERROR, "%s", msg); + peerctx->errstr = gf_strdup (msg); + goto out; + } + + ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp); + if (ret < 0) { + snprintf (msg, sizeof (msg), "Failed to decode XDR"); + gf_log ("", GF_LOG_ERROR, "%s", msg); + peerctx->errstr = gf_strdup (msg); + goto out; + } + + op_errno = rsp.op_errno; + if (-1 == rsp.op_ret) { + ret = -1; + snprintf (msg, sizeof (msg), + "Failed to get handshake ack from remote server"); + gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg); + peerctx->errstr = gf_strdup (msg); + goto out; + } + + /* TODO: this is hardcoded as of now, but I don't forsee any problems + * with this as long as we are properly handshaking operating versions + */ + peerinfo->mgmt = &gd_mgmt_prog; + peerinfo->peer = &gd_peer_prog; + + ret = default_notify (this, GF_EVENT_CHILD_UP, NULL); + + if (GD_MODE_ON == peerctx->args.mode) { + ret = glusterd_event_connected_inject (peerctx); + peerctx->args.req = NULL; + } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) { + peerctx->args.mode = GD_MODE_ON; + } else { + gf_log (this->name, GF_LOG_WARNING, "unknown mode %d", + peerctx->args.mode); + } + + glusterd_friend_sm (); + + ret = 0; +out: + + frame->local = NULL; + STACK_DESTROY (frame->root); + + if (ret != 0) + rpc_transport_disconnect (peerinfo->rpc->conn.trans); + + if (rsp.hndsk.hndsk_val) + free (rsp.hndsk.hndsk_val); + + return 0; +} + +int +glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov, + int count, void *myframe) +{ + int ret = -1; + int op_errno = EINVAL; + gf_mgmt_hndsk_rsp rsp = {0,}; + gf_mgmt_hndsk_req arg = {{0,}}; + xlator_t *this = NULL; + call_frame_t *frame = NULL; + glusterd_peerinfo_t *peerinfo = NULL; + glusterd_peerctx_t *peerctx = NULL; + dict_t *dict = NULL; + dict_t *rsp_dict = NULL; + glusterd_conf_t *conf = NULL; + char msg[1024] = {0,}; + + this = THIS; + conf = this->private; + frame = myframe; + peerctx = frame->local; + peerinfo = peerctx->peerinfo; + + if (-1 == req->rpc_status) { + ret = -1; + snprintf (msg, sizeof (msg), + "Error through RPC layer, retry again later"); + gf_log (this->name, GF_LOG_ERROR, "%s", msg); + peerctx->errstr = gf_strdup (msg); + goto out; + } + + ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp); + if (ret < 0) { + snprintf (msg, sizeof (msg), "Failed to decode management " + "handshake response"); + gf_log (this->name, GF_LOG_ERROR, "%s", msg); + peerctx->errstr = gf_strdup (msg); + goto out; + } + + GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val, + rsp.hndsk.hndsk_len, ret, op_errno, + out); + + op_errno = rsp.op_errno; + if (-1 == rsp.op_ret) { + gf_log (this->name, GF_LOG_ERROR, + "failed to get the 'versions' from peer (%s)", + req->conn->trans->peerinfo.identifier); + goto out; + } + + /* Check if peer can be part of cluster */ + ret = gd_validate_peer_op_version (this, peerinfo, dict, + &peerctx->errstr); + if (ret < 0) { + gf_log (this->name, GF_LOG_ERROR, + "failed to validate the operating version of peer (%s)", + peerinfo->hostname); + goto out; + } + + rsp_dict = dict_new (); + if (!rsp_dict) + goto out; + + ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, + "failed to set operating version in dict"); + goto out; + } + + GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val), + arg.hndsk.hndsk_len, op_errno, out); + + ret = glusterd_submit_request (peerctx->peerinfo->rpc, &arg, frame, + &gd_clnt_mgmt_hndsk_prog, + GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this, + glusterd_mgmt_hndsk_version_ack_cbk, + (xdrproc_t)xdr_gf_mgmt_hndsk_req); + +out: + if (ret) { + frame->local = NULL; + STACK_DESTROY (frame->root); + rpc_transport_disconnect (peerinfo->rpc->conn.trans); + } + + if (rsp.hndsk.hndsk_val) + free (rsp.hndsk.hndsk_val); + + if (arg.hndsk.hndsk_val) + GF_FREE (arg.hndsk.hndsk_val); + + if (dict) + dict_unref (dict); + + if (rsp_dict) + dict_unref (rsp_dict); + + return 0; +} + +int +glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx) +{ + call_frame_t *frame = NULL; + gf_mgmt_hndsk_req req = {{0,},}; + int ret = -1; + + frame = create_frame (this, this->ctx->pool); + if (!frame) + goto out; + + frame->local = peerctx; + + ret = glusterd_submit_request (peerctx->peerinfo->rpc, &req, frame, + &gd_clnt_mgmt_hndsk_prog, + GD_MGMT_HNDSK_VERSIONS, NULL, this, + glusterd_mgmt_hndsk_version_cbk, + (xdrproc_t)xdr_gf_mgmt_hndsk_req); + ret = 0; +out: + if (ret && frame) + STACK_DESTROY (frame->root); + + return ret; +} + int glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo, gf_prog_detail *prog) { - gf_prog_detail *trav = NULL; - int ret = -1; + gf_prog_detail *trav = NULL; + int ret = -1; if (!peerinfo || !prog) goto out; @@ -356,39 +837,65 @@ glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo, trav = prog; while (trav) { - /* Select 'programs' */ + ret = -1; if ((gd_mgmt_prog.prognum == trav->prognum) && (gd_mgmt_prog.progver == trav->progver)) { peerinfo->mgmt = &gd_mgmt_prog; ret = 0; } + if ((gd_peer_prog.prognum == trav->prognum) && (gd_peer_prog.progver == trav->progver)) { peerinfo->peer = &gd_peer_prog; ret = 0; } + if (ret) { gf_log ("", GF_LOG_DEBUG, "%s (%"PRId64":%"PRId64") not supported", trav->progname, trav->prognum, trav->progver); } + trav = trav->next; } if (peerinfo->mgmt) { - gf_log ("", GF_LOG_INFO, - "Using Program %s, Num (%d), Version (%d)", - peerinfo->mgmt->progname, peerinfo->mgmt->prognum, - peerinfo->mgmt->progver); + gf_log ("", GF_LOG_INFO, + "Using Program %s, Num (%d), Version (%d)", + peerinfo->mgmt->progname, peerinfo->mgmt->prognum, + peerinfo->mgmt->progver); } + if (peerinfo->peer) { - gf_log ("", GF_LOG_INFO, - "Using Program %s, Num (%d), Version (%d)", - peerinfo->peer->progname, peerinfo->peer->prognum, - peerinfo->peer->progver); + gf_log ("", GF_LOG_INFO, + "Using Program %s, Num (%d), Version (%d)", + peerinfo->peer->progname, peerinfo->peer->prognum, + peerinfo->peer->progver); } + ret = 0; +out: + return ret; + +} +static gf_boolean_t +_mgmt_hndsk_prog_present (gf_prog_detail *prog) { + gf_boolean_t ret = _gf_false; + gf_prog_detail *trav = NULL; + + GF_ASSERT (prog); + + trav = prog; + + while (trav) { + if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) && + (trav->progver == GD_MGMT_HNDSK_VERSION)) { + ret = _gf_true; + goto out; + } + trav = trav->next; + } out: return ret; } @@ -405,9 +912,11 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov, call_frame_t *frame = NULL; glusterd_peerinfo_t *peerinfo = NULL; glusterd_peerctx_t *peerctx = NULL; + glusterd_conf_t *conf = NULL; char msg[1024] = {0,}; this = THIS; + conf = this->private; frame = myframe; peerctx = frame->local; peerinfo = peerctx->peerinfo; @@ -435,6 +944,22 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov, goto out; } + if (_mgmt_hndsk_prog_present (rsp.prog)) { + gf_log (this->name, GF_LOG_DEBUG, + "Proceeding to op-version handshake with peer %s", + peerinfo->hostname); + ret = glusterd_mgmt_handshake (this, peerctx); + goto out; + } else if (conf->op_version > 1) { + ret = -1; + snprintf (msg, sizeof (msg), + "Peer %s does not support required op-version", + peerinfo->hostname); + peerctx->errstr = gf_strdup (msg); + gf_log (this->name, GF_LOG_ERROR, "%s", msg); + goto out; + } + /* Make sure we assign the proper program to peer */ ret = glusterd_set_clnt_mgmt_program (peerinfo, rsp.prog); if (ret) { @@ -454,10 +979,11 @@ glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov, peerctx->args.mode); } - glusterd_friend_sm (); - glusterd_op_sm (); + glusterd_friend_sm(); + glusterd_op_sm(); ret = 0; + out: /* don't use GF_FREE, buffer was allocated by libc */ @@ -482,8 +1008,8 @@ out: int -glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc, - glusterd_peerctx_t *peerctx) +glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc, + glusterd_peerctx_t *peerctx) { call_frame_t *frame = NULL; gf_dump_req req = {0,}; diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 6f08cc7e2..6fa490e47 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -1505,36 +1505,37 @@ out: return ret; } -int32_t -glusterd_store_uuid () +int +glusterd_store_global_info (xlator_t *this) { - glusterd_conf_t *priv = NULL; - char path[PATH_MAX] = {0,}; - int32_t ret = -1; - glusterd_store_handle_t *handle = NULL; - xlator_t *this = NULL; + int ret = -1; + glusterd_conf_t *conf = NULL; + char op_version_str[15] = {0,}; + char path[PATH_MAX] = {0,}; + glusterd_store_handle_t *handle = NULL; + char *uuid_str = NULL; - this = THIS; - priv = this->private; + conf = this->private; - snprintf (path, PATH_MAX, "%s/%s", priv->workdir, - GLUSTERD_INFO_FILE); + uuid_str = gf_strdup (uuid_utoa (MY_UUID)); + if (!uuid_str) + goto out; - if (!priv->handle) { + if (!conf->handle) { + snprintf (path, PATH_MAX, "%s/%s", conf->workdir, + GLUSTERD_INFO_FILE); ret = glusterd_store_handle_new (path, &handle); - if (ret) { gf_log (this->name, GF_LOG_ERROR, - "Unable to get store handle!"); + "Unable to get store handle"); goto out; } - priv->handle = handle; - } else { - handle = priv->handle; - } + conf->handle = handle; + } else + handle = conf->handle; - /* make glusterd's uuid available for users */ + /* These options need to be available for all users */ ret = chmod (handle->path, 0644); if (ret) { gf_log (this->name, GF_LOG_ERROR, "chmod error for %s: %s", @@ -1542,27 +1543,132 @@ glusterd_store_uuid () goto out; } - handle->fd = open (handle->path, O_RDWR | O_CREAT | O_TRUNC, 0644); + handle->fd = glusterd_store_mkstemp (handle); if (handle->fd <= 0) { ret = -1; goto out; } - ret = glusterd_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY, - uuid_utoa (MY_UUID)); + ret = glusterd_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY, + uuid_str); if (ret) { gf_log (this->name, GF_LOG_CRITICAL, "Storing uuid failed ret = %d", ret); goto out; } + snprintf (op_version_str, 15, "%d", conf->op_version); + ret = glusterd_store_save_value (handle->fd, GD_OP_VERSION_KEY, + op_version_str); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Storing op-version failed ret = %d", ret); + goto out; + } + ret = glusterd_store_rename_tmppath (handle); out: + if (ret && (handle->fd > 0)) + glusterd_store_unlink_tmppath (handle); + if (handle->fd > 0) { close (handle->fd); handle->fd = 0; } - gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret); + + if (uuid_str) + GF_FREE (uuid_str); + + if (ret) + gf_log (this->name, GF_LOG_ERROR, + "Failed to store glusterd global-info"); + + return ret; +} + +int +glusterd_retrieve_op_version (xlator_t *this, int *op_version) +{ + char *op_version_str = NULL; + glusterd_conf_t *priv = NULL; + int ret = -1; + int tmp_version = 0; + char *tmp = NULL; + char path[PATH_MAX] = {0,}; + glusterd_store_handle_t *handle = NULL; + + priv = this->private; + + if (!priv->handle) { + snprintf (path, PATH_MAX, "%s/%s", priv->workdir, + GLUSTERD_INFO_FILE); + ret = glusterd_store_handle_retrieve (path, &handle); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get store " + "handle!"); + goto out; + } + + priv->handle = handle; + } + + ret = glusterd_store_retrieve_value (priv->handle, + GD_OP_VERSION_KEY, + &op_version_str); + if (ret) { + gf_log (this->name, GF_LOG_DEBUG, + "No previous op_version present"); + goto out; + } + + tmp_version = strtol (op_version_str, &tmp, 10); + if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) { + gf_log (this->name, GF_LOG_WARNING, "invalid version number"); + goto out; + } + + *op_version = tmp_version; + + ret = 0; +out: + if (op_version_str) + GF_FREE (op_version_str); + + return ret; +} + +static int +glusterd_restore_op_version (xlator_t *this) +{ + glusterd_conf_t *conf = NULL; + int ret = 0; + int op_version = 0; + + conf = this->private; + + ret = glusterd_retrieve_op_version (this, &op_version); + if (!ret) { + if ((op_version < GD_OP_VERSION_MIN) || + (op_version > GD_OP_VERSION_MAX)) { + gf_log (this->name, GF_LOG_ERROR, + "wrong op-version (%d) retreived", op_version); + ret = -1; + goto out; + } + conf->op_version = op_version; + gf_log ("glusterd", GF_LOG_INFO, + "retrieved op-version: %d", conf->op_version); + goto out; + } + + gf_log (this->name, GF_LOG_INFO, "op-version not found in store, " + "setting it to minimum op-version : %d", GD_OP_VERSION_MIN); + + /* If op-version is missing, set it to GD_OP_VERSION_MIN */ + conf->op_version = GD_OP_VERSION_MIN; + ret = 0; +out: return ret; } @@ -1577,7 +1683,6 @@ glusterd_retrieve_uuid () priv = THIS->private; - if (!priv->handle) { snprintf (path, PATH_MAX, "%s/%s", priv->workdir, GLUSTERD_INFO_FILE); @@ -2793,8 +2898,15 @@ glusterd_restore () this = THIS; - ret = glusterd_store_retrieve_volumes (this); + ret = glusterd_restore_op_version (this); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to restore op_version"); + goto out; + } + + ret = glusterd_store_retrieve_volumes (this); if (ret) goto out; diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h index dd34250d7..1e9e599a4 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.h +++ b/xlators/mgmt/glusterd/src/glusterd-store.h @@ -94,9 +94,6 @@ glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t a int32_t glusterd_store_delete_volume (glusterd_volinfo_t *volinfo); -int32_t -glusterd_store_uuid (); - int32_t glusterd_store_handle_new (char *path, glusterd_store_handle_t **handle); @@ -134,4 +131,11 @@ glusterd_store_is_valid_brickpath (char *volname, char *brick); int32_t glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo); + +int +glusterd_retrieve_op_version (xlator_t *this, int *op_version); + +int +glusterd_store_global_info (xlator_t *this); + #endif diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c index 164009bfe..cb5ac321c 100644 --- a/xlators/mgmt/glusterd/src/glusterd.c +++ b/xlators/mgmt/glusterd/src/glusterd.c @@ -51,6 +51,7 @@ extern struct rpcsvc_program gd_svc_mgmt_prog; extern struct rpcsvc_program gd_svc_peer_prog; extern struct rpcsvc_program gd_svc_cli_prog; extern struct rpc_clnt_program gd_brick_prog; +extern struct rpcsvc_program glusterd_mgmt_hndsk_prog; rpcsvc_cbk_program_t glusterd_cbk_prog = { .progname = "Gluster Callback", @@ -58,6 +59,15 @@ rpcsvc_cbk_program_t glusterd_cbk_prog = { .progver = GLUSTER_CBK_VERSION, }; +struct rpcsvc_program *all_programs[] = { + &gd_svc_peer_prog, + &gd_svc_cli_prog, + &gd_svc_mgmt_prog, + &gluster_pmap_prog, + &gluster_handshake_prog, + &glusterd_mgmt_hndsk_prog, +}; +int rpcsvc_programs_count = (sizeof (all_programs) / sizeof (all_programs[0])); static int glusterd_opinfo_init () @@ -69,33 +79,38 @@ glusterd_opinfo_init () return ret; } + int glusterd_uuid_init () { int ret = -1; + xlator_t *this = NULL; glusterd_conf_t *priv = NULL; - priv = THIS->private; + this = THIS; + GF_ASSERT (this); + priv = this->private; ret = glusterd_retrieve_uuid (); if (ret == 0) { uuid_copy (glusterd_uuid, priv->uuid); - gf_log ("glusterd", GF_LOG_INFO, + gf_log (this->name, GF_LOG_INFO, "retrieved UUID: %s", uuid_utoa (priv->uuid)); return 0; } uuid_generate (glusterd_uuid); - gf_log ("glusterd", GF_LOG_INFO, - "generated UUID: %s", uuid_utoa (glusterd_uuid)); + gf_log (this->name, GF_LOG_INFO, "generated UUID: %s", + uuid_utoa (glusterd_uuid)); + uuid_copy (priv->uuid, glusterd_uuid); - ret = glusterd_store_uuid (); + ret = glusterd_store_global_info (this); if (ret) { - gf_log ("glusterd", GF_LOG_ERROR, - "Unable to store generated UUID"); + gf_log (this->name, GF_LOG_ERROR, + "Unable to store generated UUID"); return ret; } @@ -778,6 +793,7 @@ init (xlator_t *this) char cmd_log_filename [PATH_MAX] = {0,}; int first_time = 0; char *mountbroker_root = NULL; + int i = 0; #ifdef DEBUG char *valgrind_str = NULL; @@ -920,39 +936,16 @@ init (xlator_t *this) goto out; } - ret = glusterd_program_register (this, rpc, &gd_svc_peer_prog); - if (ret) { - goto out; - } - - ret = glusterd_program_register (this, rpc, &gd_svc_cli_prog); - if (ret) { - rpcsvc_program_unregister (rpc, &gd_svc_peer_prog); - goto out; - } - - ret = glusterd_program_register (this, rpc, &gd_svc_mgmt_prog); - if (ret) { - rpcsvc_program_unregister (rpc, &gd_svc_peer_prog); - rpcsvc_program_unregister (rpc, &gd_svc_cli_prog); - goto out; - } - - ret = glusterd_program_register (this, rpc, &gluster_pmap_prog); - if (ret) { - rpcsvc_program_unregister (rpc, &gd_svc_peer_prog); - rpcsvc_program_unregister (rpc, &gd_svc_cli_prog); - rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog); - goto out; - } + for (i = 0; i < rpcsvc_programs_count; i++) { + ret = glusterd_program_register (this, rpc, all_programs[i]); + if (ret) { + i--; + for (; i >= 0; i--) + rpcsvc_program_unregister (rpc, + all_programs[i]); - ret = glusterd_program_register (this, rpc, &gluster_handshake_prog); - if (ret) { - rpcsvc_program_unregister (rpc, &gd_svc_peer_prog); - rpcsvc_program_unregister (rpc, &gluster_pmap_prog); - rpcsvc_program_unregister (rpc, &gd_svc_cli_prog); - rpcsvc_program_unregister (rpc, &gd_svc_mgmt_prog); - goto out; + goto out; + } } conf = GF_CALLOC (1, sizeof (glusterd_conf_t), diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index b3fe1400a..0bf5beee8 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -91,6 +91,23 @@ typedef struct { gf_boolean_t running; } nodesrv_t; +#define GD_OP_VERSION_KEY "operating-version" +#define GD_MIN_OP_VERSION_KEY "minimum-operating-version" +#define GD_MAX_OP_VERSION_KEY "maxium-operating-version" + +/* Gluster versions - OP-VERSION mapping + * + * 3.3.0 - 1 + * 3.3.Next/3.Next - 2 + * + * (TODO: Change above comment once gluster version is finalised) + */ +#define GD_OP_VERSION_MIN 1 /* MIN is the fresh start op-version, mostly + should not change */ +#define GD_OP_VERSION_MAX 2 /* MAX VERSION is the maximum count in VME table, + should keep changing with introduction of newer + versions */ + typedef struct { struct _volfile_ctx *volfile; pthread_mutex_t mutex; @@ -110,15 +127,21 @@ typedef struct { gf_timer_t *timer; glusterd_sm_tr_log_t op_sm_log; struct rpc_clnt_program *gfs_mgmt; + struct list_head mount_specs; #ifdef DEBUG gf_boolean_t valgrind; #endif pthread_t brick_thread; void *hooks_priv; - xlator_t *xl; /* Should be set to 'THIS' before creating thread */ + xlator_t *xl; /* Should be set to 'THIS' before creating thread */ + + /* need for proper handshake_t */ + int op_version; /* Starts with 1 for 3.3.0 */ + } glusterd_conf_t; + typedef enum gf_brick_status { GF_BRICK_STOPPED, GF_BRICK_STARTED, @@ -556,8 +579,8 @@ glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags, glusterd_op_t op); int -glusterd_peer_handshake (xlator_t *this, struct rpc_clnt *rpc, - glusterd_peerctx_t *peerctx); +glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc, + glusterd_peerctx_t *peerctx); int glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr); -- cgit