cli,glusterd,debug/io-stats: Clear top stats

Introduces a cli command to clear "volume top" stats. The syntax for clearing
stats is
 # gluster volume top <VOLNAME> clear [nfs]

This patch also fixes io-stats xlator, to create ios_stat struct for open fop as
well, along with create and mkdir fops. This allows stats to be recorded for
files that exist already after clearing stats or restarting brick process, which
would not occur otherwise.

Change-Id: I1b87e192025be7bb8a51eaff3d09c747d05d7def
BUG: 771709
Signed-off-by: Kaushal M <kaushal@redhat.com>
Reviewed-on: http://review.gluster.com/3334
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>

4 files changed, 161 insertions, 75 deletions

diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 2dde16744e9..6c8d374ebc1 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -1672,7 +1672,7 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         char    *delimiter      = NULL;
         char    *opwords[]      = { "open", "read", "write", "opendir",
                                     "readdir", "read-perf", "write-perf",
-                                    NULL };
+                                    "clear", NULL };
         char    *w = NULL;
 
         GF_ASSERT (words);
@@ -1717,6 +1717,13 @@ cli_cmd_volume_top_parse (const char **words, int wordcount,
         } else if (strcmp (w, "write-perf") == 0) {
                 top_op = GF_CLI_TOP_WRITE_PERF;
                 perf = 1;
+        } else if (strcmp (w, "clear") == 0) {
+                ret = dict_set_int32 (dict, "clear-stats", 1);
+                if (ret) {
+                        gf_log ("cli", GF_LOG_ERROR,
+                                "Could not set clear-stats in dict");
+                        goto out;
+                }
         } else
                 GF_ASSERT (!"opword mismatch");
         ret = dict_set_int32 (dict, "top-op", (int32_t)top_op);
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index bad9351fd52..704f9dddb7d 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -1890,8 +1890,8 @@ struct cli_cmd volume_cmds[] = {
           "quota translator specific operations"},
 
          { "volume top <VOLNAME> {[open|read|write|opendir|readdir [nfs]] "
-           "|[read-perf|write-perf [nfs|{bs <size> count <count>}]]} "
-           " [brick <brick>] [list-cnt <count>]",
+           "|[read-perf|write-perf [nfs|{bs <size> count <count>}]]"
+           "|[clear [nfs]]} [brick <brick>] [list-cnt <count>]",
            cli_cmd_volume_top_cbk,
            "volume top operations"},
 
diff --git a/cli/src/cli-rpc-ops.c b/cli/src/cli-rpc-ops.c
index e54d4d219d9..9737e13f929 100644
--- a/cli/src/cli-rpc-ops.c
+++ b/cli/src/cli-rpc-ops.c
@@ -4048,7 +4048,7 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         gf_cli_rsp                        rsp   = {0,};
         int                               ret   = -1;
         dict_t                            *dict = NULL;
-        gf1_cli_stats_op                op = GF_CLI_STATS_NONE;
+        gf1_cli_stats_op                  op = GF_CLI_STATS_NONE;
         char                              key[256] = {0};
         int                               i = 0;
         int32_t                           brick_count = 0;
@@ -4056,7 +4056,7 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         int32_t                           members = 0;
         char                              *filename;
         char                              *bricks;
-        uint64_t                           value = 0;
+        uint64_t                          value = 0;
         int32_t                           j = 0;
         gf1_cli_top_op                    top_op = GF_CLI_TOP_NONE;
         uint64_t                          nr_open = 0;
@@ -4064,10 +4064,13 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         double                            throughput = 0;
         double                            time = 0;
         long int                          time_sec = 0;
-        long int                           time_usec = 0;
+        long int                          time_usec = 0;
         struct tm                         *tm = NULL;
         char                              timestr[256] = {0, };
         char                              *openfd_str = NULL;
+        gf_boolean_t                      nfs = _gf_false;
+        gf_boolean_t                      clear_stats = _gf_false;
+        int                               stats_cleared = 0;
 
         if (-1 == req->rpc_status) {
                 goto out;
@@ -4132,14 +4135,31 @@ gf_cli3_1_top_volume_cbk (struct rpc_req *req, struct iovec *iov,
         ret = dict_get_int32 (dict, key, (int32_t*)&top_op);
         if (ret)
                 goto out;
+
+        clear_stats = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+
         while (i < brick_count) {
                 i++;
                 snprintf (brick, sizeof (brick), "%d-brick", i);
                 ret = dict_get_str (dict, brick, &bricks);
                 if (ret)
                         goto out;
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret)
+
+                nfs = dict_get_str_boolean (dict, "nfs", _gf_false);
+
+                if (clear_stats) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-stats-cleared", i);
+                        ret = dict_get_int32 (dict, key, &stats_cleared);
+                        if (ret)
+                                goto out;
+                        cli_out (stats_cleared ? "Cleared stats for %s %s" :
+                                 "Failed to clear stats for %s %s",
+                                 nfs ? "NFS server on" : "brick", bricks);
+                        continue;
+                }
+
+                if (nfs)
                         cli_out ("NFS Server : %s", bricks);
                 else
                         cli_out ("Brick: %s", bricks);
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 7466f82e930..be46562029f 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -58,7 +58,7 @@ typedef enum {
         IOS_STATS_TYPE_READDIRP,
         IOS_STATS_TYPE_READ_THROUGHPUT,
         IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX,
+        IOS_STATS_TYPE_MAX
 }ios_stats_type_t;
 
 typedef enum {
@@ -1270,6 +1270,16 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         ios_fd_ctx_set (fd, this, iosfd);
 
         ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
 
         LOCK (&conf->lock);
         {
@@ -2442,6 +2452,95 @@ io_stats_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+	for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+		conf->thru_list[i].iosstats = GF_CALLOC (1,
+		                 sizeof (*conf->thru_list[i].iosstats),
+				 gf_io_stats_mt_ios_stat);
+
+	        if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+		INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+		LOCK_INIT (&conf->thru_list[i].lock);
+        }
+
+        return 0;
+}
+
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        if (list)
+                                GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        if (list)
+                                GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2509,7 +2608,6 @@ int
 init (xlator_t *this)
 {
         struct ios_conf    *conf = NULL;
-        int                 i = 0;
         char               *sys_log_str = NULL;
         int                 sys_log_level = -1;
         char               *log_str = NULL;
@@ -2546,35 +2644,9 @@ init (xlator_t *this)
         gettimeofday (&conf->cumulative.started_at, NULL);
         gettimeofday (&conf->incremental.started_at, NULL);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
-
-                if (!conf->list[i].iosstats) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                               "Out of memory");
-                        return -1;
-                }
-
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
-
-	for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-		conf->thru_list[i].iosstats = GF_CALLOC (1,
-		                 sizeof (*conf->thru_list[i].iosstats),
-				 gf_io_stats_mt_ios_stat);
-
-	        if (!conf->thru_list[i].iosstats) {
-        	        gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                	return -1;
-        	}
-
-		INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-		LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
 
         GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
@@ -2606,12 +2678,6 @@ void
 fini (xlator_t *this)
 {
         struct ios_conf *conf = NULL;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_stat_list *entry     = NULL;
-        struct ios_stat_list *tmp       = NULL;
-        struct ios_stat_list *list      = NULL;
-        struct ios_stat      *stat      = NULL;
-        int    i                        = 0;
 
         if (!this)
                 return;
@@ -2622,35 +2688,7 @@ fini (xlator_t *this)
                 return;
         this->private = NULL;
 
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
-
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
+        ios_destroy_top_stats (conf);
 
         if (conf)
                 GF_FREE(conf);
@@ -2660,7 +2698,6 @@ fini (xlator_t *this)
         return;
 }
 
-
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
@@ -2680,6 +2717,28 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         va_end (ap);
         switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                         ret = dict_set_int32 (output, "top-op", top_op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
                 ret = dict_get_int32 (dict, "top-op", &top_op);
                 if (!ret) {
                         ret = dict_get_int32 (dict, "list-cnt", &list_cnt);