From 8d4d8088c73f2ffc60b8e5c23a33e6b996fb5053 Mon Sep 17 00:00:00 2001 From: Kaushal M Date: Tue, 6 Sep 2011 17:18:20 +0530 Subject: glusterd: run 'volume top read-perf/write-perf' in different thread Runs the 'volume top read-perf/write-perf' operations in a different thread without blocking glusterd. Prvents glusterd from being unresponsive when large values of 'bs' and 'count' are given. Also increase cli timeout for top/profile commands , from 120s to 300s to allow large i/o top read-perf and write-perf to return result. Change-Id: I4b7de1d735f33643d836772db7f25133f112b75a BUG: 2720 Reviewed-on: http://review.gluster.com/375 Tested-by: Gluster Build System Reviewed-by: Shishir Gowda --- glusterfsd/src/glusterfsd-mem-types.h | 1 + glusterfsd/src/glusterfsd-mgmt.c | 352 +++++++++++++++++++++++++++++++--- glusterfsd/src/glusterfsd.h | 16 +- 3 files changed, 341 insertions(+), 28 deletions(-) (limited to 'glusterfsd') diff --git a/glusterfsd/src/glusterfsd-mem-types.h b/glusterfsd/src/glusterfsd-mem-types.h index 7f953d1ff80..a28a7b2e31b 100644 --- a/glusterfsd/src/glusterfsd-mem-types.h +++ b/glusterfsd/src/glusterfsd-mem-types.h @@ -30,6 +30,7 @@ enum gfd_mem_types_ { gfd_mt_xlator_cmdline_option_t, gfd_mt_char, gfd_mt_call_pool_t, + gfd_mt_vol_top_priv_t, gfd_mt_end }; diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c index 330b83fa36f..43d49b3f4e8 100644 --- a/glusterfsd/src/glusterfsd-mgmt.c +++ b/glusterfsd/src/glusterfsd-mgmt.c @@ -22,6 +22,7 @@ #include #include #include +#include #ifndef _CONFIG_H #define _CONFIG_H @@ -38,11 +39,12 @@ #include "protocol-common.h" #include "glusterfs3.h" #include "portmap-xdr.h" -#include "glusterd1-xdr.h" #include "xdr-generic.h" #include "glusterfsd.h" +#include "glusterfsd-mem-types.h" #include "rpcsvc.h" +#include "cli1-xdr.h" static char is_mgmt_rpc_reconnect; @@ -252,29 +254,84 @@ glusterfs_translator_info_response_send (rpcsvc_request_t *req, int ret, return ret; } +int +glusterfs_handle_translator_info_get_cont (gfd_vol_top_priv_t *priv) +{ + int ret = -1; + xlator_t *any = NULL; + xlator_t *xlator = NULL; + glusterfs_graph_t *active = NULL; + glusterfs_ctx_t *ctx = NULL; + char msg[2048] = {0,}; + dict_t *output = NULL; + dict_t *dict = NULL; + + GF_ASSERT (priv); + + dict = dict_new (); + ret = dict_unserialize (priv->xlator_req.input.input_val, + priv->xlator_req.input.input_len, &dict); + if (ret) { + gf_log ("glusterd", GF_LOG_ERROR, "Unable to unserialize dict"); + goto cont; + } + ret = dict_set_double (dict, "time", priv->time); + if (ret) + goto cont; + ret = dict_set_double (dict, "throughput", priv->throughput); + if (ret) + goto cont; + +cont: + ctx = glusterfs_ctx_get (); + GF_ASSERT (ctx); + active = ctx->active; + any = active->first; + + xlator = xlator_search_by_name (any, priv->xlator_req.name); + if (!xlator) { + snprintf (msg, sizeof (msg), "xlator %s is not loaded", + priv->xlator_req.name); + goto out; + } + + output = dict_new (); + ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output); + +out: + ret = glusterfs_translator_info_response_send (priv->req, ret, + msg, output); + + if (priv->xlator_req.name) + free (priv->xlator_req.name); + if (priv->xlator_req.input.input_val) + free (priv->xlator_req.input.input_val); + if (dict) + dict_unref (dict); + if (output) + dict_unref (output); + GF_FREE (priv); + + return ret; +} + int glusterfs_handle_translator_info_get (rpcsvc_request_t *req) { int32_t ret = -1; gd1_mgmt_brick_op_req xlator_req = {0,}; dict_t *dict = NULL; - xlator_t *xlator = NULL; - xlator_t *any = NULL; - dict_t *output = NULL; - char msg[2048] = {0}; - glusterfs_ctx_t *ctx = NULL; - glusterfs_graph_t *active = NULL; xlator_t *this = NULL; + gf1_cli_top_op top_op = 0; + int32_t blk_size = 0; + int32_t blk_count = 0; + gfd_vol_top_priv_t *priv = NULL; + pthread_t tid = -1; GF_ASSERT (req); this = THIS; GF_ASSERT (this); - ctx = glusterfs_ctx_get (); - GF_ASSERT (ctx); - - active = ctx->active; - any = active->first; if (!xdr_to_generic (req->msg[0], &xlator_req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req)) { //failed to decode msg; @@ -293,29 +350,270 @@ glusterfs_handle_translator_info_get (rpcsvc_request_t *req) goto out; } - xlator = xlator_search_by_name (any, xlator_req.name); - if (!xlator) { - snprintf (msg, sizeof (msg), "xlator %s is not loaded", - xlator_req.name); - ret = -1; + priv = GF_MALLOC (sizeof (gfd_vol_top_priv_t), gfd_mt_vol_top_priv_t); + if (!priv) { + gf_log ("glusterd", GF_LOG_ERROR, "failed to allocate memory"); goto out; } - - output = dict_new (); - ret = xlator->notify (xlator, GF_EVENT_TRANSLATOR_INFO, dict, output); + priv->xlator_req = xlator_req; + priv->req = req; + + ret = dict_get_int32 (dict, "top-op", (int32_t *)&top_op); + if ((!ret) && (GF_CLI_TOP_READ_PERF == top_op || + GF_CLI_TOP_WRITE_PERF == top_op)) { + ret = dict_get_int32 (dict, "blk-size", &blk_size); + if (ret) + goto cont; + ret = dict_get_int32 (dict, "blk-cnt", &blk_count); + if (ret) + goto cont; + priv->blk_size = blk_size; + priv->blk_count = blk_count; + if (GF_CLI_TOP_READ_PERF == top_op) { + ret = pthread_create (&tid, NULL, + glusterfs_volume_top_read_perf, + priv); + } else if ( GF_CLI_TOP_WRITE_PERF == top_op) { + ret = pthread_create (&tid, NULL, + glusterfs_volume_top_write_perf, + priv); + } + if (ret) { + gf_log ("glusterd", GF_LOG_ERROR, + "Thread create failed"); + goto cont; + } + gf_log ("glusterd", GF_LOG_DEBUG, "Created new thread with " + "tid %u", (unsigned int)tid); + goto out; + } +cont: + priv->throughput = 0; + priv->time = 0; + ret = glusterfs_handle_translator_info_get_cont (priv); out: - ret = glusterfs_translator_info_response_send (req, ret, msg, output); if (dict) dict_unref (dict); - if (xlator_req.input.input_val) - free (xlator_req.input.input_val); // malloced by xdr - if (output) - dict_unref (output); - if (xlator_req.name) - free (xlator_req.name); //malloced by xdr return ret; } +void * +glusterfs_volume_top_write_perf (void *args) +{ + int32_t fd = -1; + int32_t input_fd = -1; + char export_path[PATH_MAX]; + char *buf = NULL; + int32_t blk_size = 0; + int32_t blk_count = 0; + int32_t iter = 0; + int32_t ret = -1; + int64_t total_blks = 0; + struct timeval begin, end = {0,}; + double throughput = 0; + double time = 0; + gfd_vol_top_priv_t *priv = NULL; + + GF_ASSERT (args); + priv = (gfd_vol_top_priv_t *)args; + + blk_size = priv->blk_size; + blk_count = priv->blk_count; + + snprintf (export_path, sizeof (export_path), "%s/%s", + priv->xlator_req.name, ".gf-tmp-stats-perf"); + + fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU); + if (-1 == fd) { + ret = -1; + gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file"); + goto out; + } + + buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char); + if (!buf) { + ret = -1; + goto out; + } + + input_fd = open ("/dev/urandom", O_RDONLY); + if (-1 == input_fd) { + ret = -1; + gf_log ("glusterd",GF_LOG_ERROR, "Unable to open input file"); + goto out; + } + + gettimeofday (&begin, NULL); + for (iter = 0; iter < blk_count; iter++) { + ret = read (input_fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + ret = write (fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + total_blks += ret; + } + ret = 0; + if (total_blks != (blk_size * blk_count)) { + gf_log ("glusterd", GF_LOG_WARNING, "Error in write"); + ret = -1; + goto out; + } + + gettimeofday (&end, NULL); + time = (end.tv_sec - begin.tv_sec) * 1e6 + + (end.tv_usec - begin.tv_usec); + throughput = total_blks / time; + gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs " + "bytes written %"PRId64, throughput, time, total_blks); + +out: + priv->throughput = throughput; + priv->time = time; + + if (fd >= 0) + close (fd); + if (input_fd >= 0) + close (input_fd); + if (buf) + GF_FREE (buf); + unlink (export_path); + + (void)glusterfs_handle_translator_info_get_cont (priv); + + return NULL; +} + +void * +glusterfs_volume_top_read_perf (void *args) +{ + int32_t fd = -1; + int32_t input_fd = -1; + int32_t output_fd = -1; + char export_path[PATH_MAX]; + char *buf = NULL; + int32_t blk_size = 0; + int32_t blk_count = 0; + int32_t iter = 0; + int32_t ret = -1; + int64_t total_blks = 0; + struct timeval begin, end = {0,}; + double throughput = 0; + double time = 0; + gfd_vol_top_priv_t *priv = NULL; + + GF_ASSERT (args); + priv = (gfd_vol_top_priv_t *)args; + + blk_size = priv->blk_size; + blk_count = priv->blk_count; + + snprintf (export_path, sizeof (export_path), "%s/%s", + priv->xlator_req.name, ".gf-tmp-stats-perf"); + fd = open (export_path, O_CREAT|O_RDWR, S_IRWXU); + if (-1 == fd) { + ret = -1; + gf_log ("glusterd", GF_LOG_ERROR, "Could not open tmp file"); + goto out; + } + + buf = GF_MALLOC (blk_size * sizeof(*buf), gf_common_mt_char); + if (!buf) { + ret = -1; + gf_log ("glusterd", GF_LOG_ERROR, "Could not allocate memory"); + goto out; + } + + input_fd = open ("/dev/urandom", O_RDONLY); + if (-1 == input_fd) { + ret = -1; + gf_log ("glusterd", GF_LOG_ERROR, "Could not open input file"); + goto out; + } + + output_fd = open ("/dev/null", O_RDWR); + if (-1 == output_fd) { + ret = -1; + gf_log ("glusterd", GF_LOG_ERROR, "Could not open output file"); + goto out; + } + + for (iter = 0; iter < blk_count; iter++) { + ret = read (input_fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + ret = write (fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + } + + ret = fsync (fd); + if (ret) { + gf_log ("glusterd", GF_LOG_ERROR, "could not flush cache"); + goto out; + } + ret = lseek (fd, 0L, 0); + if (ret != 0) { + gf_log ("glusterd", GF_LOG_ERROR, + "could not seek back to start"); + ret = -1; + goto out; + } + gettimeofday (&begin, NULL); + for (iter = 0; iter < blk_count; iter++) { + ret = read (fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + ret = write (output_fd, buf, blk_size); + if (ret != blk_size) { + ret = -1; + goto out; + } + total_blks += ret; + } + ret = 0; + if ((blk_size * blk_count) != total_blks) { + ret = -1; + gf_log ("glusterd", GF_LOG_WARNING, "Error in write"); + goto out; + } + + gettimeofday (&end, NULL); + time = (end.tv_sec - begin.tv_sec) * 1e6 + + (end.tv_usec - begin.tv_usec); + throughput = total_blks / time; + gf_log ("glusterd", GF_LOG_INFO, "Throughput %.2f Mbps time %.2f secs " + "bytes read %"PRId64, throughput, time, total_blks); + +out: + priv->throughput = throughput; + priv->time = time; + + if (fd >= 0) + close (fd); + if (input_fd >= 0) + close (input_fd); + if (output_fd >= 0) + close (output_fd); + if (buf) + GF_FREE (buf); + unlink (export_path); + + (void)glusterfs_handle_translator_info_get_cont (priv); + + return NULL; +} + int glusterfs_handle_rpc_msg (rpcsvc_request_t *req) { diff --git a/glusterfsd/src/glusterfsd.h b/glusterfsd/src/glusterfsd.h index 5106559c14c..0e68f7f0ea8 100644 --- a/glusterfsd/src/glusterfsd.h +++ b/glusterfsd/src/glusterfsd.h @@ -24,7 +24,8 @@ #define _CONFIG_H #include "config.h" #endif - +#include "rpcsvc.h" +#include "glusterd1-xdr.h" #define DEFAULT_GLUSTERD_VOLFILE CONFDIR "/glusterd.vol" #define DEFAULT_CLIENT_VOLFILE CONFDIR "/glusterfs.vol" @@ -81,9 +82,22 @@ enum argp_option_keys { ARGP_USER_MAP_ROOT_KEY = 156, }; +struct _gfd_vol_top_priv_t { + rpcsvc_request_t *req; + gd1_mgmt_brick_op_req xlator_req; + int32_t blk_count; + int32_t blk_size; + double throughput; + double time; + int32_t ret; +}; +typedef struct _gfd_vol_top_priv_t gfd_vol_top_priv_t; + int glusterfs_mgmt_pmap_signout (glusterfs_ctx_t *ctx); int glusterfs_mgmt_pmap_signin (glusterfs_ctx_t *ctx); int glusterfs_volfile_fetch (glusterfs_ctx_t *ctx); void cleanup_and_exit (int signum); +void *glusterfs_volume_top_read_perf (void *args); +void *glusterfs_volume_top_write_perf (void *args); #endif /* __GLUSTERFSD_H__ */ -- cgit