From acdeed002d30209e0a058c2df0346d4f16c08994 Mon Sep 17 00:00:00 2001 From: Pavan Sondur Date: Fri, 6 Aug 2010 05:31:45 +0000 Subject: add pump xlator and changes for replace-brick Signed-off-by: Pavan Vilas Sondur Signed-off-by: Anand V. Avati BUG: 1235 (Bug for all pump/migrate commits) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1235 --- xlators/mgmt/glusterd/src/glusterd-handler.c | 55 ++++ xlators/mgmt/glusterd/src/glusterd-op-sm.c | 377 +++++++++++++++++++++++++++ xlators/mgmt/glusterd/src/glusterd-utils.c | 7 +- xlators/mgmt/glusterd/src/glusterd-utils.h | 2 + xlators/mgmt/glusterd/src/glusterd.h | 7 + xlators/mgmt/glusterd/src/glusterd3_1-mops.c | 5 + 6 files changed, 451 insertions(+), 2 deletions(-) (limited to 'xlators/mgmt') diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c index 9aa4801e99b..a8b334c3dd9 100644 --- a/xlators/mgmt/glusterd/src/glusterd-handler.c +++ b/xlators/mgmt/glusterd/src/glusterd-handler.c @@ -847,6 +847,44 @@ out: return ret; } +int +glusterd_handle_replace_brick (rpcsvc_request_t *req) +{ + int32_t ret = -1; + gf1_cli_replace_brick_req cli_req = {0,}; + dict_t *dict = NULL; + + GF_ASSERT (req); + + if (!gf_xdr_to_cli_replace_brick_req (req->msg[0], &cli_req)) { + //failed to decode msg; + req->rpc_err = GARBAGE_ARGS; + goto out; + } + + gf_log ("glusterd", GF_LOG_NORMAL, "Received replace brick req"); + + if (cli_req.bricks.bricks_len) { + /* Unserialize the dictionary */ + dict = dict_new (); + + ret = dict_unserialize (cli_req.bricks.bricks_val, + cli_req.bricks.bricks_len, + &dict); + if (ret < 0) { + gf_log ("glusterd", GF_LOG_ERROR, + "failed to " + "unserialize req-buffer to dictionary"); + goto out; + } + } + + ret = glusterd_replace_brick (req, dict); + +out: + return ret; +} + int glusterd_handle_remove_brick (rpcsvc_request_t *req) { @@ -1656,6 +1694,23 @@ glusterd_add_brick (rpcsvc_request_t *req, dict_t *dict) return ret; } +int32_t +glusterd_replace_brick (rpcsvc_request_t *req, dict_t *dict) +{ + int32_t ret = -1; + + GF_ASSERT (req); + GF_ASSERT (dict); + + glusterd_op_set_op (GD_OP_REPLACE_BRICK); + + glusterd_op_set_ctx (GD_OP_REPLACE_BRICK, dict); + + ret = glusterd_op_txn_begin (); + + return ret; +} + int32_t glusterd_remove_brick (rpcsvc_request_t *req, dict_t *dict) { diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 31e2a9e50f2..8f6861417be 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -84,6 +84,7 @@ glusterd_op_get_len (glusterd_op_t op) case GD_OP_START_BRICK: break; + case GD_OP_REPLACE_BRICK: case GD_OP_ADD_BRICK: { dict_t *dict = glusterd_op_get_ctx (op); @@ -210,6 +211,20 @@ glusterd_op_build_payload (glusterd_op_t op, gd1_mgmt_stage_op_req **req) } break; + case GD_OP_REPLACE_BRICK: + { + dict_t *dict = NULL; + dict = glusterd_op_get_ctx (op); + GF_ASSERT (dict); + ret = dict_allocate_and_serialize (dict, + &stage_req->buf.buf_val, + (size_t *)&stage_req->buf.buf_len); + if (ret) { + goto out; + } + } + break; + case GD_OP_REMOVE_BRICK: { dict_t *dict = NULL; @@ -532,6 +547,55 @@ out: return ret; } +static int +glusterd_op_stage_replace_brick (gd1_mgmt_stage_op_req *req) +{ + int ret = 0; + dict_t *dict = NULL; + char *src_brick = NULL; + char *dst_brick = NULL; + + GF_ASSERT (req); + + dict = dict_new (); + if (!dict) + goto out; + + ret = dict_unserialize (req->buf.buf_val, req->buf.buf_len, &dict); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to unserialize dict"); + goto out; + } + /* Need to do a little more error checking and validation */ + ret = dict_get_str (dict, "src-brick", &src_brick); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get src brick"); + goto out; + } + + gf_log ("", GF_LOG_DEBUG, + "src brick=%s", src_brick); + + ret = dict_get_str (dict, "dst-brick", &dst_brick); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get dest brick"); + goto out; + } + + gf_log ("", GF_LOG_DEBUG, + "dest brick=%s", dst_brick); + + ret = 0; + +out: + gf_log ("", GF_LOG_DEBUG, "Returning %d", ret); + + return ret; +} + static int glusterd_op_stage_remove_brick (gd1_mgmt_stage_op_req *req) { @@ -823,6 +887,310 @@ glusterd_op_add_brick (gd1_mgmt_stage_op_req *req) +out: + return ret; +} + +static int +replace_brick_start_dst_brick (glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *dst_brickinfo) +{ glusterd_conf_t *priv = NULL; + char cmd_str[8192] = {0,}; + char filename[PATH_MAX]; + FILE *file = NULL; + int ret; + + priv = THIS->private; + + snprintf (filename, PATH_MAX, "%s/vols/%s/replace_dst_brick.vol", + priv->workdir, volinfo->volname); + + + file = fopen (filename, "a+"); + if (!file) { + gf_log ("", GF_LOG_DEBUG, + "Open of volfile failed"); + return -1; + } + + truncate (filename, 0); + + fprintf (file, "volume src-posix\n"); + fprintf (file, "type storage/posix\n"); + fprintf (file, "option directory %s\n", + dst_brickinfo->path); + fprintf (file, "end-volume\n"); + fprintf (file, "volume %s\n", + dst_brickinfo->path); + fprintf (file, "type features/locks\n"); + fprintf (file, "subvolumes src-posix\n"); + fprintf (file, "end-volume\n"); + fprintf (file, "volume src-server\n"); + fprintf (file, "type protocol/server\n"); + fprintf (file, "option auth.addr.%s.allow *\n", + dst_brickinfo->path); + fprintf (file, "option listen-port 34034\n"); + fprintf (file, "subvolumes %s\n", + dst_brickinfo->path); + fprintf (file, "end-volume\n"); + + gf_log ("", GF_LOG_DEBUG, + "starting dst brick"); + + snprintf (cmd_str, 4096, "glusterfs -f %s/vols/%s/replace_dst_brick.vol", + priv->workdir, volinfo->volname); + + ret = system (cmd_str); + + + fclose (file); + + return 0; +} + +static int +replace_brick_spawn_brick (glusterd_volinfo_t *volinfo, dict_t *dict, + glusterd_brickinfo_t *dst_brickinfo) + +{ + + replace_brick_start_dst_brick (volinfo, dst_brickinfo); + + return 0; +} + +static int +replace_brick_generate_volfile (glusterd_volinfo_t *volinfo, + glusterd_brickinfo_t *src_brickinfo) +{ + glusterd_conf_t *priv = NULL; + FILE *file = NULL; + char filename[PATH_MAX]; + int ret; + + priv = THIS->private; + + gf_log ("", GF_LOG_DEBUG, + "Creating volfile"); + + snprintf (filename, PATH_MAX, "%s/vols/%s/replace_brick.vol", + priv->workdir, volinfo->volname); + + file = fopen (filename, "a+"); + if (!file) { + gf_log ("", GF_LOG_DEBUG, + "Open of volfile failed"); + return -1; + } + + ret = truncate (filename, 0); + ret = unlink ("/tmp/replace_brick.vol"); + + fprintf (file, "volume client/protocol\n"); + fprintf (file, "type protocol/client\n"); + fprintf (file, "option remote-host %s\n", + src_brickinfo->hostname); + fprintf (file, "option remote-subvolume %s\n", + src_brickinfo->path); + fprintf (file, "option remote-port 34034\n"); + fprintf (file, "echo end-volume\n"); + + ret = symlink(filename, "/tmp/replace_brick.vol"); + if (!ret) { + gf_log ("", GF_LOG_DEBUG, + "symlink call failed"); + return -1; + } + + fclose (file); + + return 0; +} + +static int +replace_brick_start_source_brick (glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *src_brickinfo, + glusterd_brickinfo_t *dst_brickinfo) +{ glusterd_conf_t *priv = NULL; + char filename[PATH_MAX]; + FILE *file = NULL; + char cmd_str[8192] = {0,}; + int ret; + + priv = THIS->private; + + gf_log ("", GF_LOG_DEBUG, + "Creating volfile"); + + snprintf (filename, PATH_MAX, "%s/vols/%s/replace_source_brick.vol", + priv->workdir, volinfo->volname); + + file = fopen (filename, "a+"); + if (!file) { + gf_log ("", GF_LOG_DEBUG, + "Open of volfile failed"); + return -1; + } + + ret = truncate (filename, 0); + + fprintf (file, "volume src-posix\n"); + fprintf (file, "type storage/posix\n"); + fprintf (file, "option directory %s\n", + src_brickinfo->path); + fprintf (file, "end-volume\n"); + fprintf (file, "volume locks\n"); + fprintf (file, "type features/locks\n"); + fprintf (file, "subvolumes src-posix\n"); + fprintf (file, "end-volume\n"); + fprintf (file, "volume remote-client\n"); + fprintf (file, "type protocol/client\n"); + fprintf (file, "option remote-host %s\n", + dst_brickinfo->hostname); + fprintf (file, "option remote-port 34034\n"); + fprintf (file, "option remote-subvolume %s\n", + dst_brickinfo->path); + fprintf (file, "end-volume\n"); + fprintf (file, "volume %s\n", + src_brickinfo->path); + fprintf (file, "type cluster/pump\n"); + fprintf (file, "subvolumes locks remote-client\n"); + fprintf (file, "end-volume\n"); + fprintf (file, "volume src-server\n"); + fprintf (file, "type protocol/server\n"); + fprintf (file, "option auth.addr.%s.allow *\n", + src_brickinfo->path); + fprintf (file, "option listen-port 34034\n"); + fprintf (file, "subvolumes %s\n", + src_brickinfo->path); + fprintf (file, "end-volume\n"); + + + gf_log ("", GF_LOG_DEBUG, + "starting source brick"); + + snprintf (cmd_str, 4096, "glusterfs -f %s/vols/%s/replace_source_brick.vol -l /tmp/b_log -LTRACE", + priv->workdir, volinfo->volname); + + ret = system (cmd_str); + + fclose (file); + + return 0; +} + +static int +replace_brick_mount (glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *src_brickinfo, + glusterd_brickinfo_t *dst_brickinfo, gf1_cli_replace_op op) +{ + + gf_log ("", GF_LOG_DEBUG, + "starting source brick"); + + replace_brick_start_source_brick (volinfo, src_brickinfo, + dst_brickinfo); + + return 0; +} + +static int +glusterd_op_replace_brick (gd1_mgmt_stage_op_req *req) +{ + int ret = 0; + dict_t *dict = NULL; + glusterd_volinfo_t *volinfo = NULL; + char *volname = NULL; + xlator_t *this = NULL; + glusterd_conf_t *priv = NULL; + char *src_brick = NULL; + char *dst_brick = NULL; + char *str = NULL; + + glusterd_brickinfo_t *src_brickinfo = NULL; + glusterd_brickinfo_t *dst_brickinfo = NULL; + + GF_ASSERT (req); + + this = THIS; + GF_ASSERT (this); + + priv = this->private; + GF_ASSERT (priv); + + dict = dict_new (); + if (!dict) + goto out; + + ret = dict_unserialize (req->buf.buf_val, req->buf.buf_len, &dict); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to unserialize dict"); + goto out; + } + + /* Need to do a little more error checking and validation */ + ret = dict_get_str (dict, "src-brick", &src_brick); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get src brick"); + goto out; + } + + gf_log (this->name, GF_LOG_DEBUG, + "src brick=%s", src_brick); + + ret = dict_get_str (dict, "dst-brick", &dst_brick); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get dest brick"); + goto out; + } + + gf_log (this->name, GF_LOG_DEBUG, + "dst brick=%s", dst_brick); + + ret = dict_get_str (dict, "volname", &volname); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to get volume name"); + goto out; + } + + ret = glusterd_volinfo_find (volname, &volinfo); + + if (ret) { + gf_log ("", GF_LOG_ERROR, "Unable to allocate memory"); + goto out; + } + + str = strdup (src_brick); + + ret = glusterd_brickinfo_get (str, volinfo, + &src_brickinfo); + if (ret) { + gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo"); + goto out; + } + + ret = glusterd_brickinfo_from_brick (dst_brick, &dst_brickinfo); + if (ret) { + gf_log ("", GF_LOG_DEBUG, "Unable to get dst-brickinfo"); + goto out; + } + + replace_brick_generate_volfile (volinfo, src_brickinfo); + + if (!glusterd_is_local_addr (src_brickinfo->hostname)) { + gf_log ("", GF_LOG_NORMAL, + "I AM THE SOURCE HOST"); + replace_brick_mount (volinfo, src_brickinfo, dst_brickinfo, + req->op); + } else if (!glusterd_is_local_addr (dst_brickinfo->hostname)) { + gf_log ("", GF_LOG_NORMAL, + "I AM THE DESTINATION HOST"); + replace_brick_spawn_brick (volinfo, dict, dst_brickinfo); + } + + ret = 0; + out: return ret; } @@ -1609,6 +1977,11 @@ glusterd_op_stage_validate (gd1_mgmt_stage_op_req *req) ret = glusterd_op_stage_add_brick (req); break; + case GD_OP_REPLACE_BRICK: + ret = glusterd_op_stage_replace_brick (req); + break; + + case GD_OP_REMOVE_BRICK: ret = glusterd_op_stage_remove_brick (req); break; @@ -1652,6 +2025,10 @@ glusterd_op_commit_perform (gd1_mgmt_stage_op_req *req) ret = glusterd_op_add_brick (req); break; + case GD_OP_REPLACE_BRICK: + ret = glusterd_op_replace_brick (req); + break; + case GD_OP_REMOVE_BRICK: ret = glusterd_op_remove_brick (req); break; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index 6b32bd3a0d8..5ef0ce1e5c1 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -73,7 +73,7 @@ glusterd_unset_lock_owner (uuid_t owner) return 0; } -static int32_t +int32_t glusterd_is_local_addr (char *hostname) { int32_t ret = -1; @@ -598,11 +598,14 @@ glusterd_brickinfo_from_brick (char *brick, glusterd_brickinfo_t *new_brickinfo = NULL; char *hostname = NULL; char *path = NULL; + char *tmp = NULL; GF_ASSERT (brick); GF_ASSERT (brickinfo); - hostname = strtok (brick, ":"); + tmp = strdup (brick); + + hostname = strtok (tmp, ":"); path = strtok (NULL, ":"); GF_ASSERT (hostname); diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index fc8b33ab1b9..dd97e67a68c 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -115,4 +115,6 @@ glusterd_is_cli_op_req (int32_t op); int32_t glusterd_brickinfo_get (char *brick, glusterd_volinfo_t *volinfo, glusterd_brickinfo_t **brickinfo); +int32_t +glusterd_is_local_addr (char *hostname); #endif diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h index 9f0986bb62f..368c0a6847a 100644 --- a/xlators/mgmt/glusterd/src/glusterd.h +++ b/xlators/mgmt/glusterd/src/glusterd.h @@ -249,6 +249,13 @@ glusterd_add_brick (rpcsvc_request_t *req, dict_t *dict); int glusterd_handle_add_brick (rpcsvc_request_t *req); + +int32_t +glusterd_replace_brick (rpcsvc_request_t *req, dict_t *dict); + +int +glusterd_handle_replace_brick (rpcsvc_request_t *req); + int glusterd_handle_remove_brick (rpcsvc_request_t *req); diff --git a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c index 7169121d574..86d09194e11 100644 --- a/xlators/mgmt/glusterd/src/glusterd3_1-mops.c +++ b/xlators/mgmt/glusterd/src/glusterd3_1-mops.c @@ -1164,6 +1164,10 @@ glusterd_handle_rpc_msg (rpcsvc_request_t *req) ret = glusterd_handle_add_brick (req); break; + case GD_MGMT_CLI_REPLACE_BRICK: + ret = glusterd_handle_replace_brick (req); + break; + case GD_MGMT_CLI_REMOVE_BRICK: ret = glusterd_handle_remove_brick (req); break; @@ -1203,6 +1207,7 @@ rpcsvc_actor_t glusterd1_mgmt_actors[] = { [GD_MGMT_CLI_DELETE_VOLUME] = { "DELETE_VOLUME", GD_MGMT_CLI_DELETE_VOLUME, glusterd_handle_rpc_msg, NULL, NULL}, [GD_MGMT_CLI_GET_VOLUME] = { "GET_VOLUME", GD_MGMT_CLI_GET_VOLUME, glusterd_handle_rpc_msg, NULL, NULL}, [GD_MGMT_CLI_ADD_BRICK] = { "ADD_BRICK", GD_MGMT_CLI_ADD_BRICK, glusterd_handle_rpc_msg, NULL, NULL}, + [GD_MGMT_CLI_REPLACE_BRICK] = { "REPLACE_BRICK", GD_MGMT_CLI_REPLACE_BRICK, glusterd_handle_rpc_msg, NULL, NULL}, [GD_MGMT_CLI_REMOVE_BRICK] = { "REMOVE_BRICK", GD_MGMT_CLI_REMOVE_BRICK, glusterd_handle_rpc_msg, NULL, NULL}, }; -- cgit