From e48678567a5fb01166a065f9e192b80e721e4181 Mon Sep 17 00:00:00 2001 From: Sanju Rakonde Date: Thu, 8 Nov 2018 18:50:18 +0530 Subject: glusterd: migrating profile commands to mgmt_v3 framework Current profile commands use the op_state machine framework. Porting it to use the mgmt_v3 framework. The following tests were performed on the patch: case 1: 1. On a 3 node cluster, created and started 3 volumes 2. Mounted all the three volumes and wrote some data 3. Started profile operation for all the volumes 4. Ran "gluster v status" from N1, "gluster v profile info" form N2, "gluster v profile info" from N3 simultaneously in a loop for around 10000 times 5. Didn't find any cores generated. case 2: 1. Repeat the steps 1,2 and 3 from case 1. 2. Ran "gluster v status" from N1, "gluster v profile info" form N2(terminal 1), "gluster v profile info" from N2(terminal 2) simultaneously in a loop. 3. No cores were generated. fixes: bz#1654181 Change-Id: I83044cf5aee3970ef94066c89fcc41783ed468a6 Signed-off-by: Sanju Rakonde --- xlators/mgmt/glusterd/src/glusterd-handler.c | 20 ++- xlators/mgmt/glusterd/src/glusterd-mgmt.c | 233 +++++++++++++++++++++++++-- xlators/mgmt/glusterd/src/glusterd-mgmt.h | 4 + xlators/mgmt/glusterd/src/glusterd-op-sm.c | 4 +- xlators/mgmt/glusterd/src/glusterd-op-sm.h | 6 + 5 files changed, 244 insertions(+), 23 deletions(-) (limited to 'xlators') diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index db58b3af8cf..1da6d27e18a 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -27,6 +27,7 @@ #include "glusterd-sm.h" #include "glusterd-op-sm.h" #include "glusterd-utils.h" +#include "glusterd-mgmt.h" #include "glusterd-server-quorum.h" #include "glusterd-store.h" #include "glusterd-locks.h" @@ -3031,10 +3032,13 @@ __glusterd_handle_cli_profile_volume(rpcsvc_request_t *req) 0, }; xlator_t *this = NULL; + glusterd_conf_t *conf = NULL; GF_ASSERT(req); this = THIS; GF_ASSERT(this); + conf = this->private; + GF_VALIDATE_OR_GOTO(this->name, conf, out); ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req); if (ret < 0) { @@ -3075,12 +3079,20 @@ __glusterd_handle_cli_profile_volume(rpcsvc_request_t *req) goto out; } - ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str)); + if (conf->op_version < GD_OP_VERSION_6_0) { + gf_msg_debug(this->name, 0, + "The cluster is operating at " + "version less than %d. Falling back " + "to op-sm framework.", + GD_OP_VERSION_6_0); + ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str)); + glusterd_friend_sm(); + glusterd_op_sm(); + } else { + ret = glusterd_mgmt_v3_initiate_profile_phases(req, cli_op, dict); + } out: - glusterd_friend_sm(); - glusterd_op_sm(); - free(cli_req.dict.dict_val); if (ret) { diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c index ee358808236..eb3022612be 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c @@ -19,6 +19,7 @@ #include "glusterd-locks.h" #include "glusterd-mgmt.h" #include "glusterd-op-sm.h" +#include "glusterd-server-quorum.h" #include "glusterd-volgen.h" #include "glusterd-store.h" #include "glusterd-snapshot-utils.h" @@ -210,6 +211,16 @@ gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, } break; + case GD_OP_PROFILE_VOLUME: + ret = glusterd_op_stage_stats_volume(dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_WARNING, 0, + GD_MSG_PRE_VALIDATION_FAIL, + "prevalidation failed for profile operation."); + goto out; + } + break; + case GD_OP_MAX_OPVERSION: ret = 0; break; @@ -247,6 +258,17 @@ gd_mgmt_v3_brick_op_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, } break; } + case GD_OP_PROFILE_VOLUME: { + ret = gd_brick_op_phase(op, rsp_dict, dict, op_errstr); + if (ret) { + gf_log(this->name, GF_LOG_WARNING, + "%s brickop " + "failed", + gd_op_list[op]); + goto out; + } + break; + } default: break; } @@ -382,6 +404,15 @@ gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr, } break; } + case GD_OP_PROFILE_VOLUME: { + ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL, + "commit failed for volume profile operation."); + goto out; + } + break; + } default: break; @@ -819,6 +850,7 @@ glusterd_pre_validate_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp) case GD_OP_DETACH_TIER_STATUS: case GD_OP_TIER_START_STOP: case GD_OP_REMOVE_TIER_BRICK: + case GD_OP_PROFILE_VOLUME: break; case GD_OP_MAX_OPVERSION: break; @@ -1004,6 +1036,15 @@ glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict, goto out; } + if (op == GD_OP_PROFILE_VOLUME) { + ret = glusterd_validate_quorum(this, op, req_dict, op_errstr); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET, + "Server quorum not met. Rejecting operation."); + goto out; + } + } + /* Pre Validation on local node */ ret = gd_mgmt_v3_pre_validate_fn(op, req_dict, op_errstr, rsp_dict, op_errno); @@ -1122,7 +1163,8 @@ glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict, case GD_OP_ADD_BRICK: case GD_OP_REPLACE_BRICK: case GD_OP_RESET_BRICK: - case GD_OP_ADD_TIER_BRICK: { + case GD_OP_ADD_TIER_BRICK: + case GD_OP_PROFILE_VOLUME: { ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname); if (ret) { gf_msg(this->name, GF_LOG_CRITICAL, errno, @@ -1271,12 +1313,11 @@ out: } int -glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *req_dict, char **op_errstr, - uint32_t txn_generation) +glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *rsp_dict, dict_t *req_dict, + char **op_errstr, uint32_t txn_generation) { int32_t ret = -1; int32_t peer_cnt = 0; - dict_t *rsp_dict = NULL; glusterd_peerinfo_t *peerinfo = NULL; struct syncargs args = {0}; uuid_t peer_uuid = {0}; @@ -1291,13 +1332,6 @@ glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *req_dict, char **op_errstr, GF_ASSERT(req_dict); GF_ASSERT(op_errstr); - rsp_dict = dict_new(); - if (!rsp_dict) { - gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, - "Failed to create response dictionary"); - goto out; - } - /* Perform brick op on local node */ ret = gd_mgmt_v3_brick_op_fn(op, req_dict, op_errstr, rsp_dict); @@ -1320,11 +1354,8 @@ glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *req_dict, char **op_errstr, goto out; } - dict_unref(rsp_dict); - rsp_dict = NULL; - /* Sending brick op req to other nodes in the cluster */ - gd_syncargs_init(&args, NULL); + gd_syncargs_init(&args, rsp_dict); synctask_barrier_init((&args)); peer_cnt = 0; @@ -2056,6 +2087,172 @@ out: return ret; } +int32_t +glusterd_mgmt_v3_initiate_profile_phases(rpcsvc_request_t *req, + glusterd_op_t op, dict_t *dict) +{ + int32_t ret = -1; + int32_t op_ret = -1; + dict_t *req_dict = NULL; + dict_t *tmp_dict = NULL; + glusterd_conf_t *conf = NULL; + char *op_errstr = NULL; + xlator_t *this = NULL; + gf_boolean_t is_acquired = _gf_false; + uuid_t *originator_uuid = NULL; + uint32_t txn_generation = 0; + uint32_t op_errno = 0; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(req); + GF_ASSERT(dict); + conf = this->private; + GF_ASSERT(conf); + + /* Save the peer list generation */ + txn_generation = conf->generation; + cmm_smp_rmb(); + /* This read memory barrier makes sure that this assignment happens here + * only and is not reordered and optimized by either the compiler or the + * processor. + */ + + /* Save the MY_UUID as the originator_uuid. This originator_uuid + * will be used by is_origin_glusterd() to determine if a node + * is the originator node for a command. */ + originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t); + if (!originator_uuid) { + ret = -1; + goto out; + } + + gf_uuid_copy(*originator_uuid, MY_UUID); + ret = dict_set_bin(dict, "originator_uuid", originator_uuid, + sizeof(uuid_t)); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Failed to set originator_uuid."); + GF_FREE(originator_uuid); + goto out; + } + + /* Marking the operation as complete synctasked */ + ret = dict_set_int32(dict, "is_synctasked", _gf_true); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, + "Failed to set synctasked flag."); + goto out; + } + + /* Use a copy at local unlock as cli response will be sent before + * the unlock and the volname in the dict might be removed */ + tmp_dict = dict_new(); + if (!tmp_dict) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, + "Unable to create dict"); + goto out; + } + dict_copy(dict, tmp_dict); + + /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */ + ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno, + &is_acquired, txn_generation); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL, + "mgmt_v3 lockdown failed."); + goto out; + } + + /* BUILD PAYLOAD */ + ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, + LOGSTR_BUILD_PAYLOAD, gd_op_list[op]); + if (op_errstr == NULL) + gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD); + goto out; + } + + /* PRE-COMMIT VALIDATE PHASE */ + ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno, + txn_generation); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL, + "Pre Validation Failed"); + goto out; + } + + /* BRICK-OPS */ + ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr, + txn_generation); + if (ret) { + gf_log(this->name, GF_LOG_ERROR, "Brick Op Failed"); + goto out; + } + + /* COMMIT OP PHASE */ + ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno, + txn_generation); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL, + "Commit Op Failed"); + goto out; + } + + /* POST-COMMIT VALIDATE PHASE */ + /* As of now, post_validate is not trying to cleanup any failed + commands. So as of now, I am sending 0 (op_ret as 0). + */ + ret = glusterd_mgmt_v3_post_validate(op, 0, dict, req_dict, &op_errstr, + txn_generation); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL, + "Post Validation Failed"); + goto out; + } + + ret = 0; +out: + op_ret = ret; + /* UNLOCK PHASE FOR PEERS*/ + (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr, + is_acquired, txn_generation); + + /* LOCAL VOLUME(S) UNLOCK */ + if (is_acquired) { + /* Trying to release multiple mgmt_v3 locks */ + ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID); + if (ret) { + gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL, + "Failed to release mgmt_v3 locks on localhost"); + op_ret = ret; + } + } + + if (op_ret && (op_errno == 0)) + op_errno = EG_INTRNL; + + if (op != GD_OP_MAX_OPVERSION) { + /* SEND CLI RESPONSE */ + glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, + op_errstr); + } + + if (req_dict) + dict_unref(req_dict); + + if (tmp_dict) + dict_unref(tmp_dict); + + if (op_errstr) { + GF_FREE(op_errstr); + op_errstr = NULL; + } + + return 0; +} + int32_t glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict) @@ -2398,7 +2595,8 @@ glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op, goto out; } - ret = glusterd_mgmt_v3_brick_op(op, req_dict, &op_errstr, txn_generation); + ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr, + txn_generation); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL, "Brick Ops Failed"); @@ -2458,7 +2656,8 @@ unbarrier: goto out; } - ret = glusterd_mgmt_v3_brick_op(op, req_dict, &op_errstr, txn_generation); + ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr, + txn_generation); if (ret) { gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL, diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h index 9b57f4cb833..6bd211965cf 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h +++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h @@ -35,6 +35,10 @@ int32_t glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict); +int32_t +glusterd_mgmt_v3_initiate_profile_phases(rpcsvc_request_t *req, + glusterd_op_t op, dict_t *dict); + int32_t glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op, dict_t *dict); diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 0e5b75bf5f0..fd516be1eda 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -2002,7 +2002,7 @@ out: return ret; } -static int +int glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr) { int ret = -1; @@ -3252,7 +3252,7 @@ glusterd_remove_profile_volume_options(glusterd_volinfo_t *volinfo) dict_del_sizen(volinfo->dict, VKEY_DIAG_CNT_FOP_HITS); } -static int +int glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict) { int ret = -1; diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h index 4fcaff1c8ba..fccc2c22f15 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h @@ -311,4 +311,10 @@ glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr); int32_t glusterd_tier_op(xlator_t *this, void *data); + +int +glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict); + +int +glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr); #endif -- cgit