From 5dda1d4e00b8235656b7e4e8de35f599bf033148 Mon Sep 17 00:00:00 2001 From: Vijaikumar M Date: Mon, 5 May 2014 12:43:26 +0530 Subject: glusterd/snapshot: Execute lvm snapshots in parallel Back-end LVM Snapshot is executed parallely as synop task This helps is gaining performance when there are more bricks in a node. This patch also removes unwanted logs printed in snapshot cleanup Change-Id: I3174cb4547ebb670eca37a98eb9d75ecb0672a90 BUG: 1061685 Signed-off-by: Vijaikumar M Reviewed-on: http://review.gluster.org/7461 Tested-by: Gluster Build System Reviewed-by: Rajesh Joseph Reviewed-by: Krishnan Parthasarathi Tested-by: Krishnan Parthasarathi --- xlators/mgmt/glusterd/src/glusterd-mem-types.h | 3 +- xlators/mgmt/glusterd/src/glusterd-snapshot.c | 262 ++++++++++++++++++++----- xlators/mgmt/glusterd/src/glusterd-store.c | 8 +- xlators/mgmt/glusterd/src/glusterd-utils.c | 45 +++-- xlators/mgmt/glusterd/src/glusterd-utils.h | 3 +- 5 files changed, 248 insertions(+), 73 deletions(-) (limited to 'xlators/mgmt/glusterd/src') diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h index e6f6a0333a1..95908cf62fc 100644 --- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h +++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h @@ -69,7 +69,8 @@ typedef enum gf_gld_mem_types_ { gf_gld_mt_int = gf_common_mt_end + 53, gf_gld_mt_snap_t = gf_common_mt_end + 54, gf_gld_mt_missed_snapinfo_t = gf_common_mt_end + 55, - gf_gld_mt_end = gf_common_mt_end + 56, + gf_gld_mt_snap_create_args_t = gf_common_mt_end + 56, + gf_gld_mt_end = gf_common_mt_end + 57, } gf_gld_mem_types_t; #endif diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c index e52809f3e80..65b0fb12cd5 100644 --- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c +++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c @@ -63,6 +63,18 @@ #include "lvm-defaults.h" char snap_mount_folder[PATH_MAX]; +struct snap_create_args_ { + xlator_t *this; + dict_t *dict; + dict_t *rsp_dict; + glusterd_volinfo_t *snap_vol; + glusterd_brickinfo_t *brickinfo; + struct syncargs *args; + int32_t volcount; + int32_t brickcount; + int32_t brickorder; +}; +typedef struct snap_create_args_ snap_create_args_t; /* This function is called to get the device path of the snap lvm. Usually if /dev/mapper/- is the device for the lvm, @@ -1921,13 +1933,14 @@ out: int32_t glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol) { - char *mnt_pt = NULL; - struct mntent *entry = NULL; - int32_t brick_count = -1; - int32_t ret = -1; - glusterd_brickinfo_t *brickinfo = NULL; - xlator_t *this = NULL; - FILE *mtab = NULL; + char *mnt_pt = NULL; + struct mntent *entry = NULL; + struct mntent save_entry = {0,}; + int32_t brick_count = -1; + int32_t ret = -1; + glusterd_brickinfo_t *brickinfo = NULL; + xlator_t *this = NULL; + char buff [PATH_MAX] = ""; this = THIS; GF_ASSERT (this); @@ -1988,7 +2001,8 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol) goto out; } - entry = glusterd_get_mnt_entry_info (mnt_pt, mtab); + entry = glusterd_get_mnt_entry_info (mnt_pt, buff, + sizeof (buff), &save_entry); if (!entry) { gf_log (this->name, GF_LOG_WARNING, "getting the mount" " entry for the brick %s:%s of the snap %s " @@ -2001,8 +2015,6 @@ glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol) ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo, mnt_pt, entry->mnt_fsname); - if (mtab) - endmntent (mtab); if (ret) { gf_log (this->name, GF_LOG_ERROR, "failed to " "remove the snapshot %s (%s)", @@ -2052,8 +2064,8 @@ glusterd_snap_volume_remove (dict_t *rsp_dict, "brick for volume %s", snap_vol->volname); save_ret = ret; - /* Continue to cleaning up the snap in case of error - if force flag is enabled */ + /* Don't clean up the snap on error when + force flag is disabled */ if (!force) goto out; } @@ -2156,16 +2168,14 @@ glusterd_snap_remove (dict_t *rsp_dict, list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) { ret = glusterd_snap_volume_remove (rsp_dict, snap_vol, remove_lvm, force); - if (ret) { + if (ret && !force) { + /* Don't clean up the snap on error when + force flag is disabled */ gf_log(this->name, GF_LOG_WARNING, "Failed to remove " "volinfo %s for snap %s", snap_vol->volname, snap->snapname); save_ret = ret; - - /* Continue to cleaning up the snap in case of error - if force flag is enabled */ - if (!force) - goto out; + goto out; } } @@ -4619,9 +4629,11 @@ glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict) ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true); if (ret) { - gf_log (this->name, GF_LOG_ERROR, "removing the snap %s failed", + /* Ignore failure as this is a cleanup of half cooked + snapshot */ + gf_log (this->name, GF_LOG_DEBUG, "removing the snap %s failed", name); - goto out; + ret = 0; } name = NULL; @@ -4676,6 +4688,175 @@ out: return ret; } +int +glusterd_take_brick_snapshot_task (void *opaque) +{ + int ret = 0; + snap_create_args_t *snap_args = NULL; + char key[PATH_MAX] = ""; + + GF_ASSERT (opaque); + + snap_args = (snap_create_args_t*) opaque; + THIS = snap_args->this; + + ret = glusterd_take_brick_snapshot (snap_args->dict, + snap_args->snap_vol, + snap_args->brickinfo, + snap_args->volcount, + snap_args->brickorder); + + if (ret) { + gf_log (THIS->name, GF_LOG_ERROR, "Failed to " + "take backend snapshot for brick " + "%s:%s volume(%s)", snap_args->brickinfo->hostname, + snap_args->brickinfo->path, snap_args->snap_vol->volname); + } + + snprintf (key, sizeof (key), "snap-vol%d.brick%d.status", + snap_args->volcount, snap_args->brickorder); + if (dict_set_int32 (snap_args->rsp_dict, key, (ret)?0:1)) { + gf_log (THIS->name, GF_LOG_ERROR, "failed to " + "add %s to dict", key); + ret = -1; + goto out; + } + +out: + return ret; +} + +int32_t +glusterd_take_brick_snapshot_cbk (int ret, call_frame_t *frame, void *opaque) +{ + snap_create_args_t *snap_args = NULL; + struct syncargs *args = NULL; + + GF_ASSERT (opaque); + + snap_args = (snap_create_args_t*) opaque; + args = snap_args->args; + + if (ret) + args->op_ret = ret; + + GF_FREE (opaque); + synctask_barrier_wake(args); + return 0; +} + +int32_t +glusterd_schedule_brick_snapshot (dict_t *dict, dict_t *rsp_dict, + glusterd_snap_t *snap) +{ + int ret = -1; + int32_t volcount = 0; + int32_t brickcount = 0; + int32_t brickorder = 0; + int32_t taskcount = 0; + char key[PATH_MAX] = ""; + xlator_t *this = NULL; + glusterd_volinfo_t *snap_vol = NULL; + glusterd_brickinfo_t *brickinfo = NULL; + struct syncargs args = {0}; + snap_create_args_t *snap_args = NULL; + + this = THIS; + GF_ASSERT(this); + GF_ASSERT(dict); + GF_ASSERT(snap); + + synctask_barrier_init ((&args)); + list_for_each_entry (snap_vol, &snap->volumes, vol_list) { + volcount++; + brickcount = 0; + brickorder = 0; + list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) { + snprintf (key, sizeof(key) - 1, + "snap-vol%d.brick%d.order", volcount, + brickcount); + ret = dict_set_int32 (rsp_dict, key, brickorder); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, + "Failed to set %s", key); + goto out; + } + + if ((uuid_compare (brickinfo->uuid, MY_UUID)) || + (brickinfo->snap_status == -1)) { + if (!uuid_compare (brickinfo->uuid, MY_UUID)) { + brickcount++; + snprintf (key, sizeof (key), + "snap-vol%d.brick%d.status", + volcount, brickorder); + ret = dict_set_int32 (rsp_dict, key, 0); + if (ret) { + gf_log (this->name, + GF_LOG_ERROR, + "failed to add %s to " + "dict", key); + goto out; + } + } + brickorder++; + continue; + } + + snap_args = GF_CALLOC (1, sizeof (*snap_args), + gf_gld_mt_snap_create_args_t); + if (!snap_args) { + ret = -1; + goto out; + } + + + snap_args->this = this; + snap_args->dict = dict; + snap_args->rsp_dict = rsp_dict; + snap_args->snap_vol = snap_vol; + snap_args->brickinfo = brickinfo; + snap_args->volcount = volcount; + snap_args->brickcount = brickcount; + snap_args->brickorder = brickorder; + snap_args->args = &args; + + ret = synctask_new (this->ctx->env, + glusterd_take_brick_snapshot_task, + glusterd_take_brick_snapshot_cbk, + NULL, snap_args); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to " + "spawn task for snapshot create"); + GF_FREE (snap_args); + goto out; + } + taskcount++; + brickcount++; + brickorder++; + } + + snprintf (key, sizeof (key), "snap-vol%d_brickcount", volcount); + ret = dict_set_int64 (rsp_dict, key, brickcount); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "failed to " + "add %s to dict", key); + goto out; + } + } + synctask_barrier_wait ((&args), taskcount); + taskcount = 0; + + if (args.op_ret) + gf_log (this->name, GF_LOG_ERROR, "Failed to create snapshot"); + + ret = args.op_ret; +out: + if (ret && taskcount) + synctask_barrier_wait ((&args), taskcount); + + return ret; +} + int32_t glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr, dict_t *rsp_dict) @@ -4683,7 +4864,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr, int ret = -1; int64_t i = 0; int64_t volcount = 0; - int32_t brickcount = 0; char *snapname = NULL; char *volname = NULL; char *tmp_name = NULL; @@ -4692,7 +4872,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr, glusterd_snap_t *snap = NULL; glusterd_volinfo_t *origin_vol = NULL; glusterd_volinfo_t *snap_vol = NULL; - glusterd_brickinfo_t *brickinfo = NULL; glusterd_conf_t *priv = NULL; this = THIS; @@ -4756,9 +4935,6 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr, goto out; } - /* TODO: Create a stub where the bricks are - added parallely by worker threads so that - the snap creating happens parallely. */ snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict, rsp_dict, i); if (!snap_vol) { @@ -4768,32 +4944,20 @@ glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr, goto out; } } + ret = dict_set_int64 (rsp_dict, "volcount", volcount); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount"); + goto out; + } - volcount=0; - list_for_each_entry (snap_vol, &snap->volumes, vol_list) { - volcount++; - brickcount = 0; - list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) { - if ((uuid_compare (brickinfo->uuid, MY_UUID)) || - (brickinfo->snap_status == -1)) { - brickcount++; - continue; - } - ret = glusterd_take_brick_snapshot (dict, - snap_vol, - brickinfo, - volcount, - brickcount); - if (ret) { - gf_log (THIS->name, GF_LOG_ERROR, "Failed to " - "take backend snapshot for brick " - "%s:%s volume(%s)", brickinfo->hostname, - brickinfo->path, snap_vol->volname); - goto out; - } - brickcount++; - } + ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap); + if (ret) { + gf_log (this->name, GF_LOG_ERROR, "Failed to take backend " + "snapshot %s", snap->snapname); + goto out; + } + list_for_each_entry (snap_vol, &snap->volumes, vol_list) { snap_vol->status = GLUSTERD_STATUS_STARTED; ret = glusterd_store_volinfo (snap_vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT); diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c index 5b1d80e84aa..ca6d9454826 100644 --- a/xlators/mgmt/glusterd/src/glusterd-store.c +++ b/xlators/mgmt/glusterd/src/glusterd-store.c @@ -2897,9 +2897,10 @@ out: int32_t glusterd_mount_brick_paths (char *brick_mount_path, char *device_path) { - FILE *mtab = NULL; int32_t ret = -1; runner_t runner = {0, }; + char buff [PATH_MAX] = {0, }; + struct mntent save_entry = {0, }; struct mntent *entry = NULL; xlator_t *this = NULL; glusterd_conf_t *priv = NULL; @@ -2913,7 +2914,8 @@ glusterd_mount_brick_paths (char *brick_mount_path, char *device_path) GF_ASSERT (priv); /* Check if the brick_mount_path is already mounted */ - entry = glusterd_get_mnt_entry_info (brick_mount_path, mtab); + entry = glusterd_get_mnt_entry_info (brick_mount_path, buff, + sizeof (buff), &save_entry); if (entry) { gf_log (this->name, GF_LOG_INFO, "brick_mount_path (%s) already mounted.", @@ -2949,8 +2951,6 @@ glusterd_mount_brick_paths (char *brick_mount_path, char *device_path) } out: - if (mtab) - endmntent (mtab); gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret); return ret; } diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c index dd4f84265db..1134d29dc5b 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.c +++ b/xlators/mgmt/glusterd/src/glusterd-utils.c @@ -6981,15 +6981,21 @@ glusterd_add_inode_size_to_dict (dict_t *dict, int count) } struct mntent * -glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab) +glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen, + struct mntent *entry_ptr) { - struct mntent *entry = NULL; + struct mntent *entry = NULL; + FILE *mtab = NULL; + + GF_ASSERT (mnt_pt); + GF_ASSERT (buff); + GF_ASSERT (entry_ptr); mtab = setmntent (_PATH_MOUNTED, "r"); if (!mtab) goto out; - entry = getmntent (mtab); + entry = getmntent_r (mtab, entry_ptr, buff, buflen); while (1) { if (!entry) @@ -6998,10 +7004,13 @@ glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab) if (!strcmp (entry->mnt_dir, mnt_pt) && strcmp (entry->mnt_type, "rootfs")) break; - entry = getmntent (mtab); + entry = getmntent_r (mtab, entry_ptr, buff, buflen); } out: + if (NULL != mtab) { + endmntent (mtab); + } return entry; } @@ -7011,9 +7020,10 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo, { int ret = -1; char key[1024] = {0}; + char buff [PATH_MAX] = {0}; char base_key[1024] = {0}; + struct mntent save_entry = {0}; char *mnt_pt = NULL; - FILE *mtab = NULL; struct mntent *entry = NULL; snprintf (base_key, sizeof (base_key), "brick%d", count); @@ -7022,7 +7032,8 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo, if (ret) goto out; - entry = glusterd_get_mnt_entry_info (mnt_pt, mtab); + entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff), + &save_entry); if (!entry) { ret = -1; goto out; @@ -7052,8 +7063,6 @@ glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo, out: GF_FREE (mnt_pt); - if (mtab) - endmntent (mtab); return ret; } @@ -7064,8 +7073,9 @@ glusterd_get_brick_mount_device (char *brick_path) int ret = -1; char *mnt_pt = NULL; char *device = NULL; - FILE *mtab = NULL; + char buff [PATH_MAX] = ""; struct mntent *entry = NULL; + struct mntent save_entry = {0,}; xlator_t *this = NULL; this = THIS; @@ -7079,7 +7089,8 @@ glusterd_get_brick_mount_device (char *brick_path) goto out; } - entry = glusterd_get_mnt_entry_info (mnt_pt, mtab); + entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff), + &save_entry); if (NULL == entry) { gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry " "for %s mount path", mnt_pt); @@ -7090,10 +7101,6 @@ glusterd_get_brick_mount_device (char *brick_path) device = gf_strdup (entry->mnt_fsname); out: - if (NULL != mtab) { - endmntent (mtab); - } - return device; } @@ -10362,7 +10369,8 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src) ret = dict_get_int64 (src, snapbrckcnt, &brick_count); if (ret) { gf_log (this->name, GF_LOG_TRACE, - "No bricks for this volume in this dict"); + "No bricks for this volume in this dict (%s)", + snapbrckcnt); continue; } @@ -10374,7 +10382,8 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src) ret = dict_get_int64 (src, snapbrckord, &brick_order); if (ret) { gf_log (this->name, GF_LOG_ERROR, - "Failed to get brick order"); + "Failed to get brick order (%s)", + snapbrckord); goto out; } @@ -10384,14 +10393,14 @@ glusterd_merge_brick_status (dict_t *dst, dict_t *src) ret = dict_get_int32 (src, key, &brick_online); if (ret) { gf_log (this->name, GF_LOG_ERROR, "failed to " - "get the brick status"); + "get the brick status (%s)", key); goto out; } ret = dict_set_int32 (dst, key, brick_online); if (ret) { gf_log (this->name, GF_LOG_ERROR, "failed to " - "set the brick status"); + "set the brick status (%s)", key); goto out; } brick_online = 0; diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h index 14f6ce4d08a..122969ec252 100644 --- a/xlators/mgmt/glusterd/src/glusterd-utils.h +++ b/xlators/mgmt/glusterd/src/glusterd-utils.h @@ -696,7 +696,8 @@ char* glusterd_get_brick_mount_device (char *brick_path); struct mntent * -glusterd_get_mnt_entry_info (char *mnt_pt, FILE *mtab); +glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen, + struct mntent *entry_ptr); int glusterd_get_brick_root (char *path, char **mount_point); -- cgit