logging enhancements

* per translator loglevel introduced, if set, it will override process
  wide log level.
* with extended attribute 'trusted.glusterfs.<xlator-name>.set-log-level'
  with the value being '<LOGLEVEL>', one can change log level of particular
  translator.
* with extended attribute 'trusted.glusterfs.syslog' with the value
  '<BOOLEAN>', one can enable disable syslog for gluster

Signed-off-by: Amar Tumballi <amar@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 601 (Enable changing 'log-level' of GlusterFS process at runtime)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=601

7 files changed, 208 insertions, 27 deletions

diff --git a/libglusterfs/src/logging.c b/libglusterfs/src/logging.c
index 000c2ccd34f..2d3bb4c8f25 100644
--- a/libglusterfs/src/logging.c
+++ b/libglusterfs/src/logging.c
@@ -35,39 +35,73 @@
 #include "logging.h"
 #include "defaults.h"
 
+#ifdef GF_LINUX_HOST_OS
+#include <syslog.h>
+#endif
+
+
 static pthread_mutex_t  logfile_mutex;
 static char            *filename = NULL;
 static uint8_t          logrotate = 0;
 
 static FILE            *logfile = NULL;
 static gf_loglevel_t    loglevel = GF_LOG_MAX;
+static int              gf_log_syslog = 0;
 
 gf_loglevel_t           gf_log_loglevel; /* extern'd */
 FILE                   *gf_log_logfile;
 
 
-void 
+void
 gf_log_logrotate (int signum)
 {
 	logrotate = 1;
 }
 
+void
+gf_log_enable_syslog (void)
+{
+        gf_log_syslog = 1;
+}
+
+void
+gf_log_disable_syslog (void)
+{
+        gf_log_syslog = 0;
+}
 
-gf_loglevel_t 
+gf_loglevel_t
 gf_log_get_loglevel (void)
 {
 	return loglevel;
 }
 
-
 void
 gf_log_set_loglevel (gf_loglevel_t level)
 {
-	gf_log_loglevel = loglevel = level;
+        gf_log_loglevel = loglevel = level;
 }
 
 
-void 
+gf_loglevel_t
+gf_log_get_xl_loglevel (void *this)
+{
+        xlator_t *xl = this;
+        if (!xl)
+                return 0;
+        return xl->loglevel;
+}
+
+void
+gf_log_set_xl_loglevel (void *this, gf_loglevel_t level)
+{
+        xlator_t *xl = this;
+        if (!xl)
+                return;
+        xl->loglevel = level;
+}
+
+void
 gf_log_fini (void)
 {
 	pthread_mutex_destroy (&logfile_mutex);
@@ -99,6 +133,12 @@ gf_log_init (const char *file)
 		return -1;
 	}
 
+#ifdef GF_LINUX_HOST_OS
+        /* For the 'syslog' output. one can grep 'GlusterFS' in syslog
+           for serious logs */
+        openlog ("GlusterFS", LOG_PID, LOG_DAEMON);
+#endif
+
 	gf_log_logfile = logfile;
 
 	return 0;
@@ -162,23 +202,37 @@ _gf_log (const char *domain, const char *file, const char *function, int line,
         char        *msg  = NULL;
         size_t       len  = 0;
         int          ret  = 0;
+        xlator_t    *this = NULL;
+        gf_loglevel_t xlator_loglevel = 0;
+
+        this = THIS;
+
+        xlator_loglevel = this->loglevel;
+        if (xlator_loglevel == 0)
+                xlator_loglevel = loglevel;
+
+        if (level > xlator_loglevel)
+                goto out;
 
 	static char *level_strings[] = {"",  /* NONE */
+                                        "M", /* EMERGENCY */
+                                        "A", /* ALERT */
 					"C", /* CRITICAL */
 					"E", /* ERROR */
 					"W", /* WARNING */
-					"N", /* NORMAL */
+					"N", /* NOTICE */
+                                        "I", /* INFO/NORMAL */
 					"D", /* DEBUG */
                                         "T", /* TRACE */
 					""};
-  
+
 	if (!domain || !file || !function || !fmt) {
-		fprintf (stderr, 
-			 "logging: %s:%s():%d: invalid argument\n", 
+		fprintf (stderr,
+			 "logging: %s:%s():%d: invalid argument\n",
 			 __FILE__, __PRETTY_FUNCTION__, __LINE__);
 		return -1;
 	}
-  
+
 	if (!logfile) {
 		fprintf (stderr, "no logfile set\n");
 		return (-1);
@@ -206,7 +260,7 @@ log:
 
 	tm    = localtime (&tv.tv_sec);
 
-	if (level > loglevel) {
+	if (level > xlator_loglevel) {
 		goto out;
 	}
 
@@ -241,13 +295,21 @@ log:
 
                 len = strlen (str1);
                 msg = GF_MALLOC (len + strlen (str2) + 1, gf_common_mt_char);
-                
+
                 strcpy (msg, str1);
                 strcpy (msg + len, str2);
 
 		fprintf (logfile, "%s\n", msg);
 		fflush (logfile);
+
+#ifdef GF_LINUX_HOST_OS
+                /* We want only serious log in 'syslog', not our debug
+                   and trace logs */
+                if (gf_log_syslog && level && (level <= GF_LOG_ERROR))
+                        syslog ((level-1), "%s\n", msg);
+#endif
 	}
+
 unlock:
 	pthread_mutex_unlock (&logfile_mutex);
 
diff --git a/libglusterfs/src/logging.h b/libglusterfs/src/logging.h
index fdb24dab9f6..f727bfe421d 100644
--- a/libglusterfs/src/logging.h
+++ b/libglusterfs/src/logging.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2010 Gluster, Inc. <http://www.gluster.com>
    This file is part of GlusterFS.
 
    GlusterFS is free software; you can redistribute it and/or modify
@@ -44,13 +44,28 @@
 #define GF_PRI_BLKSIZE     PRId32
 #define GF_PRI_SIZET       "zu"
 
+#if 0
+/* Syslog definitions :-) */
+#define LOG_EMERG   0           /* system is unusable */
+#define LOG_ALERT   1           /* action must be taken immediately */
+#define LOG_CRIT    2           /* critical conditions */
+#define LOG_ERR     3           /* error conditions */
+#define LOG_WARNING 4           /* warning conditions */
+#define LOG_NOTICE  5           /* normal but significant condition */
+#define LOG_INFO    6           /* informational */
+#define LOG_DEBUG   7           /* debug-level messages */
+#endif
+
 typedef enum {
 	GF_LOG_NONE,
+        GF_LOG_EMERG,
+        GF_LOG_ALERT,
 	GF_LOG_CRITICAL,   /* fatal errors */
 	GF_LOG_ERROR,      /* major failures (not necessarily fatal) */
 	GF_LOG_WARNING,    /* info about normal operation */
+        GF_LOG_NOTICE,
 	GF_LOG_INFO,       /* Normal information */
-#define GF_LOG_NORMAL GF_LOG_INFO
+#define GF_LOG_NORMAL      GF_LOG_INFO
 	GF_LOG_DEBUG,      /* internal errors */
         GF_LOG_TRACE,      /* full trace of operation */
 } gf_loglevel_t;
@@ -60,9 +75,8 @@ typedef enum {
 extern gf_loglevel_t gf_log_loglevel;
 
 #define gf_log(dom, levl, fmt...) do {					\
-		if (levl <= gf_log_loglevel)				\
-			_gf_log (dom, __FILE__, __FUNCTION__, __LINE__, \
-				 levl, ##fmt);				\
+                _gf_log (dom, __FILE__, __FUNCTION__, __LINE__,         \
+                         levl, ##fmt);                                  \
 		if (0) {						\
 			printf (fmt);					\
 		}							\
@@ -73,8 +87,8 @@ extern gf_loglevel_t gf_log_loglevel;
                 gf_log (args);                                                \
         }
 
-			
-void 
+
+void
 gf_log_logrotate (int signum);
 
 int gf_log_init (const char *filename);
@@ -90,10 +104,12 @@ gf_log_from_client (const char *msg, char *identifier);
 void gf_log_lock (void);
 void gf_log_unlock (void);
 
-gf_loglevel_t 
-gf_log_get_loglevel (void);
-void 
-gf_log_set_loglevel (gf_loglevel_t level);
+void gf_log_disable_syslog (void);
+void gf_log_enable_syslog (void);
+gf_loglevel_t gf_log_get_loglevel (void);
+void gf_log_set_loglevel (gf_loglevel_t level);
+gf_loglevel_t gf_log_get_xl_loglevel (void *xl);
+void gf_log_set_xl_loglevel (void *xl, gf_loglevel_t level);
 
 #define GF_DEBUG(xl, format, args...) \
 	gf_log ((xl)->name, GF_LOG_DEBUG, format, ##args)
diff --git a/libglusterfs/src/xlator.h b/libglusterfs/src/xlator.h
index ef79c354fcc..208f7a64a56 100644
--- a/libglusterfs/src/xlator.h
+++ b/libglusterfs/src/xlator.h
@@ -75,8 +75,6 @@ typedef int32_t (*event_notify_fn_t) (xlator_t *this, int32_t event, void *data,
 #include "globals.h"
 #include "iatt.h"
 
-
-
 struct _loc {
 	const char *path;
 	const char *name;
@@ -85,6 +83,7 @@ struct _loc {
 	inode_t    *parent;
 };
 
+
 typedef int32_t (*fop_getspec_cbk_t) (call_frame_t *frame,
 				      void *cookie,
 				      xlator_t *this,
@@ -816,6 +815,8 @@ struct _xlator {
 	int32_t           (*mem_acct_init) (xlator_t *this);
 	event_notify_fn_t notify;
 
+        gf_loglevel_t    loglevel;   /* Log level for translator */
+
         /* for latency measurement */
         fop_latency_t latencies[GF_FOP_MAXVALUE];
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index be8c95b584e..e3337c7d985 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -801,7 +801,7 @@ glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
         gf_log ("", GF_LOG_NORMAL, "Stopping glusterfs running in pid: %d",
                 pid);
 
-        ret = kill (pid, SIGQUIT);
+        ret = kill (pid, SIGTERM);
 
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to kill pid %d", pid);
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index ea5b5ae38ad..ac8295b96de 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -2290,6 +2290,15 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
         }
 #endif
 
+        /* Check if the command is for changing the log
+           level of process or specific xlator */
+        ret = is_gf_log_command (this, name, value);
+        if (ret >= 0) {
+                send_fuse_err (this, finh, ret);
+                GF_FREE (finh);
+                return;
+        }
+
         GET_STATE (this, finh, state);
         state->size = fsi->size;
         ret = fuse_loc_fill (&state->loc, state, finh->nodeid, 0, NULL);
@@ -2311,6 +2320,7 @@ fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
                         "%"PRIu64": SETXATTR dict allocation failed",
                         finh->unique);
 
+                send_fuse_err (this, finh, ENOMEM);
                 free_fuse_state (state);
                 return;
         }
@@ -2749,7 +2759,6 @@ fuse_setlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
         return;
 }
 
-
 static void
 fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
 {
@@ -3001,6 +3010,9 @@ fuse_thread_proc (void *data)
         priv->msg0_len_p = &iov_in[0].iov_len;
 
         for (;;) {
+                /* THIS has to be reset here */
+                THIS = this;
+
                 if (priv->init_recvd)
                         fuse_graph_sync (this);
 
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index 8165053e06f..afefd815a14 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -27,6 +27,7 @@
 #include <dirent.h>
 #include <sys/mount.h>
 #include <sys/time.h>
+#include <fnmatch.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -273,5 +274,6 @@ xlator_t *fuse_state_subvol (fuse_state_t *state);
 xlator_t *fuse_active_subvol (xlator_t *fuse);
 inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
 int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
+int is_gf_log_command (xlator_t *this, const char *name, char *value);
 
 #endif /* _GF_FUSE_BRIDGE_H_ */
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index d478d014db0..9a6b13514aa 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -298,3 +298,91 @@ gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa)
         fa->flags      = 0;
 #endif
 }
+
+
+int
+is_gf_log_command (xlator_t *this, const char *name, char *value)
+{
+        fuse_private_t *priv        = NULL;
+        xlator_t       *trav        = NULL;
+        char            key[1024]   = {0,};
+        int             ret         = -1;
+        int             log_level   = -1;
+        gf_boolean_t    syslog_flag = 0;
+
+        priv = this->private;
+
+        if (!strcmp ("trusted.glusterfs.syslog", name)) {
+                ret = gf_string2boolean (value, &syslog_flag);
+                if (ret) {
+                        ret = EOPNOTSUPP;
+                        goto out;
+                }
+                if (syslog_flag)
+                        gf_log_enable_syslog ();
+                else
+                        gf_log_disable_syslog ();
+
+                goto out;
+        }
+
+        if (fnmatch ("trusted.glusterfs*set-log-level", name, FNM_NOESCAPE))
+                goto out;
+
+        if (!strcasecmp (value, "CRITICAL")) {
+                log_level = GF_LOG_CRITICAL;
+        } else if (!strcasecmp (value, "ERROR")) {
+                log_level = GF_LOG_ERROR;
+        } else if (!strcasecmp (value, "WARNING")) {
+                log_level = GF_LOG_WARNING;
+        } else if (!strcasecmp (value, "INFO")) {
+                log_level = GF_LOG_INFO;
+        } else if (!strcasecmp (value, "DEBUG")) {
+                log_level = GF_LOG_DEBUG;
+        } else if (!strcasecmp (value, "TRACE")) {
+                log_level = GF_LOG_TRACE;
+        } else if (!strcasecmp (value, "NONE")) {
+                log_level = GF_LOG_NONE;
+        }
+
+        if (log_level == -1) {
+                ret = EOPNOTSUPP;
+                goto out;
+        }
+
+        /* Some crude way to change the log-level of process */
+        if (!strcmp (name, "trusted.glusterfs.set-log-level")) {
+                /* */
+                gf_log ("glusterfs", gf_log_get_loglevel(),
+                        "setting log level to %d (old-value=%d)",
+                        log_level, gf_log_get_loglevel());
+                gf_log_set_loglevel (log_level);
+                ret = 0;
+                goto out;
+        }
+        if (!strcmp (name, "trusted.glusterfs.fuse.set-log-level")) {
+                /* */
+                gf_log (this->name, gf_log_get_xl_loglevel (this),
+                        "setting log level to %d (old-value=%d)",
+                        log_level, gf_log_get_xl_loglevel (this));
+                gf_log_set_xl_loglevel (this, log_level);
+                ret = 0;
+                goto out;
+        }
+
+        trav = priv->active_subvol;
+        while (trav) {
+                snprintf (key, 1024, "trusted.glusterfs.%s.set-log-level",
+                          trav->name);
+                if (fnmatch (name, key, FNM_NOESCAPE) == 0) {
+                        gf_log (trav->name, gf_log_get_xl_loglevel (trav),
+                                "setting log level to %d (old-value=%d)",
+                                log_level, gf_log_get_xl_loglevel (trav));
+                        gf_log_set_xl_loglevel (trav, log_level);
+                        ret = 0;
+                }
+                trav = trav->next;
+        }
+out:
+        return ret;
+}