core/statedump: statedump enahancements

* append timestamp to the statedump filename to prevent old files
  getting over written
* Add start and end markers to statedump to indicate beginning and
  finishing of statedump information
* Make glusterfs take options through /tmp/glusterdump.options file and
  treating those options with higher prioriry
* do not dump the entire inode table in the statedump. Instead just dump
  the ltable and the fdtable

Signed-off-by: Raghavendra Bhat <raghavendra@redhat.com>

5 files changed, 259 insertions, 45 deletions

diff --git a/libglusterfs/src/fd.c b/libglusterfs/src/fd.c
index b0b75a5ad3e..aafda5ecc27 100644
--- a/libglusterfs/src/fd.c
+++ b/libglusterfs/src/fd.c
@@ -960,6 +960,13 @@ fd_dump (fd_t *fd, char *prefix)
         gf_proc_dump_write("pid", "%llu", fd->pid);
         gf_proc_dump_write("refcount", "%d", fd->refcount);
         gf_proc_dump_write("flags", "%d", fd->flags);
+
+        if (fd->inode) {
+                gf_proc_dump_build_key (key, "inode", NULL);
+                gf_proc_dump_add_section(key);
+                inode_dump (fd->inode, key);
+        }
+
 }
 
 
diff --git a/libglusterfs/src/statedump.c b/libglusterfs/src/statedump.c
index 899d8ef2b1b..8e0b7aeca24 100644
--- a/libglusterfs/src/statedump.c
+++ b/libglusterfs/src/statedump.c
@@ -60,8 +60,9 @@ gf_proc_dump_open (char *dump_dir, char *brickname)
         char path[PATH_MAX] = {0,};
         int  dump_fd = -1;
 
-        snprintf (path, sizeof (path), "%s/%s.%d.dump", (dump_dir ?
-                  dump_dir : "/tmp"), brickname, getpid());
+        snprintf (path, sizeof (path), "%s/%s.%d.dump.%"PRIu64,
+                  (dump_dir ? dump_dir : "/tmp"), brickname, getpid(),
+                  (uint64_t) time (NULL));
 
         dump_fd = open (path, O_CREAT|O_RDWR|O_TRUNC|O_APPEND, 0600);
         if (dump_fd < 0)
@@ -79,6 +80,45 @@ gf_proc_dump_close (void)
         gf_dump_fd = -1;
 }
 
+static int
+gf_proc_dump_set_path (char *dump_options_file)
+{
+        int     ret = -1;
+        FILE    *fp = NULL;
+        char    buf[256];
+        char    *key = NULL, *value = NULL;
+        char    *saveptr = NULL;
+
+        fp = fopen (dump_options_file, "r");
+        if (!fp)
+                goto out;
+
+        ret = fscanf (fp, "%s", buf);
+
+        while (ret != EOF) {
+                key = strtok_r (buf, "=", &saveptr);
+                if (!key) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+
+                value = strtok_r (NULL, "=", &saveptr);
+
+                if (!value) {
+                        ret = fscanf (fp, "%s", buf);
+                        continue;
+                }
+                if (!strcmp (key, "path")) {
+                        dump_options.dump_path = gf_strdup (value);
+                        break;
+                }
+        }
+
+out:
+        if (fp)
+                fclose (fp);
+        return ret;
+}
 
 int
 gf_proc_dump_add_section (char *key, ...)
@@ -395,8 +435,6 @@ gf_proc_dump_xlator_info (xlator_t *top)
                     (trav->itable)) {
                         snprintf (itable_key, 1024, "%d.%s.itable",
                                   ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
                 }
 
                 if (!trav->dumpops) {
@@ -430,24 +468,20 @@ static void
 gf_proc_dump_oldgraph_xlator_info (xlator_t *top)
 {
         xlator_t        *trav = NULL;
-        glusterfs_ctx_t *ctx = NULL;
-        char             itable_key[1024] = {0,};
 
         if (!top)
                 return;
 
-        ctx = glusterfs_ctx_get ();
-
         trav = top;
         while (trav) {
                 gf_proc_dump_xlator_mem_info_only_in_use (trav);
 
                 if (GF_PROC_DUMP_IS_XL_OPTION_ENABLED (inode) &&
                     (trav->itable)) {
-                        snprintf (itable_key, 1024, "%d.%s.itable",
-                                  ctx->graph_id, trav->name);
-
-                        inode_table_dump (trav->itable, itable_key);
+                        /*TODO: dump inode table info if necessary by
+                          printing the graph id (taken by glusterfs_cbtx_t)
+                          in the key
+                        */
                 }
 
                 if (!trav->dumpops) {
@@ -488,6 +522,44 @@ gf_proc_dump_enable_all_options ()
         return 0;
 }
 
+gf_boolean_t
+is_gf_proc_dump_all_disabled ()
+{
+        gf_boolean_t all_disabled = _gf_true;
+
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_mem, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_iobuf, all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.dump_callpool, all_disabled,
+                                   out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_priv,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inode,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fd,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_inodectx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_fdctx,
+                                   all_disabled, out);
+        GF_CHECK_DUMP_OPTION_ENABLED (dump_options.xl_options.dump_history,
+                                   all_disabled, out);
+
+out:
+        return all_disabled;
+}
+
+/* These options are dumped by default if /tmp/glusterdump.options
+   file exists and it is emtpty
+*/
+static int
+gf_proc_dump_enable_default_options ()
+{
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_mem, _gf_true);
+        GF_PROC_DUMP_SET_OPTION (dump_options.dump_callpool, _gf_true);
+
+        return 0;
+}
+
 static int
 gf_proc_dump_disable_all_options ()
 {
@@ -566,28 +638,43 @@ gf_proc_dump_options_init ()
         int     ret = -1;
         FILE    *fp = NULL;
         char    buf[256];
-        char    dumpbuf[GF_DUMP_MAX_BUF_LEN];
         char    *key = NULL, *value = NULL;
         char    *saveptr = NULL;
         char    dump_option_file[PATH_MAX];
 
+        /* glusterd will create a file /tmp/glusterdump.<pid>.options and
+           sets the statedump options for the process and the file is removed
+           after the statedump is taken. Direct issue of SIGUSR1 does not have
+           mechanism for considering the statedump options. So to have a way
+           of configuring the statedump of all the glusterfs processes through
+           both cli command and SIGUSR1, /tmp/glusterdump.options file
+           is searched and the options mentioned in it are given the higher
+           priority.
+        */
         snprintf (dump_option_file, sizeof (dump_option_file),
-                  "/tmp/glusterdump.%d.options", getpid ());
-
+                  "/tmp/glusterdump.options");
         fp = fopen (dump_option_file, "r");
-
         if (!fp) {
-                //ENOENT, return success
-                (void) gf_proc_dump_enable_all_options ();
-                return 0;
+                snprintf (dump_option_file, sizeof (dump_option_file),
+                          "/tmp/glusterdump.%d.options", getpid ());
+
+                fp = fopen (dump_option_file, "r");
+
+                if (!fp) {
+                        //ENOENT, return success
+                        (void) gf_proc_dump_enable_all_options ();
+                        return 0;
+                }
         }
 
         (void) gf_proc_dump_disable_all_options ();
 
+        // swallow the errors if setting statedump file path is failed.
+        ret = gf_proc_dump_set_path (dump_option_file);
+
         ret = fscanf (fp, "%s", buf);
 
         while (ret != EOF) {
-
                 key = strtok_r (buf, "=", &saveptr);
                 if (!key) {
                         ret = fscanf (fp, "%s", buf);
@@ -601,13 +688,15 @@ gf_proc_dump_options_init ()
                         continue;
                 }
 
-                snprintf (dumpbuf, sizeof (dumpbuf), "[Debug]:key=%s, value=%s\n",key,value);
-                ret = write (gf_dump_fd, dumpbuf, strlen (dumpbuf));
-
                 gf_proc_dump_parse_set_option (key, value);
-
         }
 
+        if (is_gf_proc_dump_all_disabled ())
+                (void) gf_proc_dump_enable_default_options ();
+
+        if (fp)
+                fclose (fp);
+
         return 0;
 }
 
@@ -619,6 +708,9 @@ gf_proc_dump_info (int signum)
         glusterfs_ctx_t   *ctx  = NULL;
         glusterfs_graph_t *trav = NULL;
         char               brick_name[PATH_MAX] = {0,};
+        struct timeval     tv   = {0,};
+        char timestr[256]       = {0,};
+        char sign_string[512]   = {0,};
 
         gf_proc_dump_lock ();
 
@@ -631,14 +723,37 @@ gf_proc_dump_info (int signum)
         } else
                 strncpy (brick_name, "glusterdump", sizeof (brick_name));
 
-        ret = gf_proc_dump_open (ctx->statedump_path, brick_name);
+        ret = gf_proc_dump_options_init ();
         if (ret < 0)
                 goto out;
 
-        ret = gf_proc_dump_options_init ();
+                if (dump_options.dump_path)
+                ret = gf_proc_dump_open (dump_options.dump_path, brick_name);
+        else
+                ret = gf_proc_dump_open (ctx->statedump_path, brick_name);
         if (ret < 0)
                 goto out;
 
+        //continue even though gettimeofday() has failed
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                strftime (timestr, sizeof timestr, "%Y-%m-%d %H:%M:%S",
+                          localtime (&tv.tv_sec));
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "DUMP-START-TIME: %s\n",
+                  timestr);
+
+        //swallow the errors of write for start and end marker
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
+        memset (sign_string, 0, sizeof (sign_string));
+        memset (timestr, 0, sizeof (timestr));
+        memset (&tv, 0, sizeof (tv));
+
         if (GF_PROC_DUMP_IS_OPTION_ENABLED (mem)) {
                 gf_proc_dump_mem_info ();
                 gf_proc_dump_mempool_info (ctx);
@@ -670,9 +785,24 @@ gf_proc_dump_info (int signum)
                 i++;
         }
 
+        ret = gettimeofday (&tv, NULL);
+        if (0 == ret) {
+                strftime (timestr, sizeof timestr, "%Y-%m-%d %H:%M:%s",
+                          localtime (&tv.tv_sec));
+                snprintf (timestr + strlen (timestr),
+                          sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, tv.tv_usec);
+        }
+
+        snprintf (sign_string, sizeof (sign_string), "\nDUMP-END-TIME: %s",
+                  timestr);
+        ret = write (gf_dump_fd, sign_string, strlen (sign_string));
+
 out:
         if (gf_dump_fd != -1)
                 gf_proc_dump_close ();
+        GF_FREE (dump_options.dump_path);
+        dump_options.dump_path = NULL;
         gf_proc_dump_unlock ();
 
         return;
diff --git a/libglusterfs/src/statedump.h b/libglusterfs/src/statedump.h
index dc56bda0cbb..e8d304e66d4 100644
--- a/libglusterfs/src/statedump.h
+++ b/libglusterfs/src/statedump.h
@@ -31,6 +31,7 @@ typedef struct gf_dump_options_ {
         gf_boolean_t            dump_iobuf;
         gf_boolean_t            dump_callpool;
         gf_dump_xl_options_t    xl_options; //options for all xlators
+        char                    *dump_path;
 } gf_dump_options_t;
 
 extern gf_dump_options_t dump_options;
@@ -55,6 +56,14 @@ void _gf_proc_dump_build_key (char *key, const char *prefix, char *fmt,...)
 
 #define GF_PROC_DUMP_SET_OPTION(opt,val) opt = val
 
+#define GF_CHECK_DUMP_OPTION_ENABLED(option_dump, var, label)      \
+        do {                                                    \
+                if (option_dump == _gf_true) {                  \
+                        var = _gf_false;                        \
+                        goto label;                             \
+                }                                               \
+        } while (0);
+
 void gf_proc_dump_init();
 
 void gf_proc_dump_fini(void);
diff --git a/xlators/protocol/server/src/server.c b/xlators/protocol/server/src/server.c
index f51dfb9c5ef..4f6527fa46e 100644
--- a/xlators/protocol/server/src/server.c
+++ b/xlators/protocol/server/src/server.c
@@ -297,6 +297,81 @@ out:
         return ret;
 }
 
+void
+ltable_dump (server_connection_t *trav)
+{
+        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+        struct _locker *locker = NULL;
+        char    locker_data[GF_MAX_LOCK_OWNER_LEN] = {0,};
+        int     count = 0;
+
+        gf_proc_dump_build_key(key,
+                               "conn","bound_xl.ltable.inodelk.%s",
+                               trav->bound_xl->name);
+        gf_proc_dump_add_section(key);
+
+        list_for_each_entry (locker, &trav->ltable->inodelk_lockers, lockers) {
+                count++;
+                gf_proc_dump_write("volume", "%s", locker->volume);
+                if (locker->fd) {
+                        gf_proc_dump_write("fd", "%p", locker->fd);
+                        gf_proc_dump_write("gfid", "%s",
+                                           uuid_utoa (locker->fd->inode->gfid));
+                } else {
+                        gf_proc_dump_write("fd", "%s", locker->loc.path);
+                        gf_proc_dump_write("gfid", "%s",
+                                           uuid_utoa (locker->loc.inode->gfid));
+                }
+                gf_proc_dump_write("pid", "%d", locker->pid);
+                gf_proc_dump_write("lock length", "%d", locker->owner.len);
+                lkowner_unparse (&locker->owner, locker_data,
+                                 locker->owner.len);
+                gf_proc_dump_write("lock owner", "%s", locker_data);
+                memset (locker_data, 0, sizeof (locker_data));
+
+                gf_proc_dump_build_key (key, "inode", "%d", count);
+                gf_proc_dump_add_section (key);
+                if (locker->fd)
+                        inode_dump (locker->fd->inode, key);
+                else
+                        inode_dump (locker->loc.inode, key);
+        }
+
+        count = 0;
+        locker = NULL;
+        gf_proc_dump_build_key(key,
+                               "conn","bound_xl.ltable.entrylk.%s",
+                               trav->bound_xl->name);
+        gf_proc_dump_add_section(key);
+
+        list_for_each_entry (locker, &trav->ltable->entrylk_lockers,
+                             lockers) {
+                count++;
+                gf_proc_dump_write("volume", "%s", locker->volume);
+                if (locker->fd) {
+                        gf_proc_dump_write("fd", "%p", locker->fd);
+                        gf_proc_dump_write("gfid", "%s",
+                                           uuid_utoa (locker->fd->inode->gfid));
+                } else {
+                        gf_proc_dump_write("fd", "%s", locker->loc.path);
+                        gf_proc_dump_write("gfid", "%s",
+                                           uuid_utoa (locker->loc.inode->gfid));
+                }
+                gf_proc_dump_write("pid", "%d", locker->pid);
+                gf_proc_dump_write("lock length", "%d", locker->owner.len);
+                lkowner_unparse (&locker->owner, locker_data, locker->owner.len);
+                gf_proc_dump_write("lock data", "%s", locker_data);
+                memset (locker_data, 0, sizeof (locker_data));
+
+                gf_proc_dump_build_key (key, "inode", "%d", count);
+                gf_proc_dump_add_section (key);
+                if (locker->fd)
+                        inode_dump (locker->fd->inode, key);
+                else
+                        inode_dump (locker->loc.inode, key);
+        }
+}
+
 int
 server_priv_to_dict (xlator_t *this, dict_t *dict)
 {
@@ -461,7 +536,6 @@ server_inode (xlator_t *this)
         char                 key[GF_DUMP_MAX_BUF_LEN];
         int                  i = 1;
         int                  ret = -1;
-        xlator_t             *prev_bound_xl = NULL;
 
         GF_VALIDATE_OR_GOTO ("server", this, out);
 
@@ -477,31 +551,24 @@ server_inode (xlator_t *this)
                 goto out;
 
         list_for_each_entry (trav, &conf->conns, list) {
-                if (trav->bound_xl && trav->bound_xl->itable) {
-                        /* Presently every brick contains only one
-                         * bound_xl for all connections. This will lead
-                         * to duplicating of the inode lists, if listing
-                         * is done for every connection. This simple check
-                         * prevents duplication in the present case. If
-                         * need arises the check can be improved.
-                         */
-                        if (trav->bound_xl == prev_bound_xl)
-                                continue;
-                        prev_bound_xl = trav->bound_xl;
-
+                ret = pthread_mutex_trylock (&trav->lock);
+                if (!ret)
+                {
                         gf_proc_dump_build_key(key,
-                                               "conn","%d.bound_xl.%s",
-                                               i, trav->bound_xl->name);
-                        inode_table_dump(trav->bound_xl->itable,key);
+                                               "conn","%d.ltable", i);
+                        gf_proc_dump_add_section(key);
+                        ltable_dump (trav);
                         i++;
-                }
+                        pthread_mutex_unlock (&trav->lock);
+                }else
+                        continue;
         }
         pthread_mutex_unlock (&conf->mutex);
 
         ret = 0;
 out:
         if (ret)
-                gf_proc_dump_write ("Unable to dump the inode table",
+                gf_proc_dump_write ("Unable to dump the lock table",
                                     "(Lock acquisition failed) %s",
                                     this?this->name:"server");
 
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 94ceafd10cc..b975588a5f9 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -54,7 +54,6 @@ struct _locker {
 struct _lock_table {
         struct list_head  inodelk_lockers;
         struct list_head  entrylk_lockers;
-        size_t            count;
 };
 
 /* private structure per connection (transport object)
@@ -214,4 +213,6 @@ server_submit_reply (call_frame_t *frame, rpcsvc_request_t *req, void *arg,
 int gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict);
 int gf_server_check_getxattr_cmd (call_frame_t *frame, const char *name);
 
+void ltable_dump (server_connection_t *conn);
+
 #endif /* !_SERVER_H */