summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--extras/glusterfs-logrotate9
-rw-r--r--libglusterfs/src/common-utils.c260
-rw-r--r--libglusterfs/src/common-utils.h21
-rw-r--r--libglusterfs/src/glusterfs.h6
-rw-r--r--libglusterfs/src/mem-types.h2
-rw-r--r--libglusterfs/src/stack.h1
-rw-r--r--tests/basic/fop-sampling.t61
-rw-r--r--xlators/debug/io-stats/src/io-stats-mem-types.h2
-rw-r--r--xlators/debug/io-stats/src/io-stats.c494
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-set.c15
-rw-r--r--xlators/nfs/server/src/nfs-fops.c1
-rw-r--r--xlators/nfs/server/src/nfs.c15
-rw-r--r--xlators/nfs/server/src/nfs.h5
-rw-r--r--xlators/protocol/server/src/server-helpers.c7
14 files changed, 864 insertions, 35 deletions
diff --git a/extras/glusterfs-logrotate b/extras/glusterfs-logrotate
index e3319afaa96..575c0eee771 100644
--- a/extras/glusterfs-logrotate
+++ b/extras/glusterfs-logrotate
@@ -26,3 +26,12 @@
/usr/bin/killall -HUP glusterfsd > /dev/null 2>&1 || true
endscript
}
+
+/var/log/glusterfs/samples/*.samp {
+ daily
+ rotate 3
+ sharedscripts
+ missingok
+ compress
+ delaycompress
+}
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 2dcd54f1829..a89e120c0fb 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -34,6 +34,7 @@
#if defined(GF_BSD_HOST_OS) || defined(GF_DARWIN_HOST_OS)
#include <sys/sysctl.h>
#endif
+#include <libgen.h>
#include "compat-errno.h"
#include "logging.h"
@@ -210,7 +211,7 @@ out:
}
/**
- * gf_resolve_parent_path -- Given a path, returns an allocated string
+ * gf_resolve_path_parent -- Given a path, returns an allocated string
* containing the parent's path.
* @path: Path to parse
* @return: The parent path if found, NULL otherwise
@@ -359,6 +360,135 @@ err:
return -1;
}
+/**
+ * gf_dnscache_init -- Initializes a dnscache struct and sets the ttl
+ * to the specified value in the parameter.
+ *
+ * @ttl: the TTL in seconds
+ * @return: SUCCESS: Pointer to an allocated dnscache struct
+ * FAILURE: NULL
+ */
+struct dnscache *
+gf_dnscache_init (time_t ttl)
+{
+ struct dnscache *cache = GF_MALLOC (sizeof (*cache),
+ gf_common_mt_dnscache);
+ cache->cache_dict = NULL;
+ cache->ttl = ttl;
+ return cache;
+}
+
+/**
+ * gf_dnscache_entry_init -- Initialize a dnscache entry
+ *
+ * @return: SUCCESS: Pointer to an allocated dnscache entry struct
+ * FAILURE: NULL
+ */
+struct dnscache_entry *
+gf_dnscache_entry_init ()
+{
+ struct dnscache_entry *entry = GF_CALLOC (1, sizeof (*entry),
+ gf_common_mt_dnscache_entry);
+ return entry;
+}
+
+/**
+ * gf_dnscache_entry_deinit -- Free memory used by a dnscache entry
+ *
+ * @entry: Pointer to deallocate
+ */
+void
+gf_dnscache_entry_deinit (struct dnscache_entry *entry)
+{
+ GF_FREE (entry->ip);
+ GF_FREE (entry->fqdn);
+ GF_FREE (entry);
+}
+
+/**
+ * gf_rev_dns_lookup -- Perform a reverse DNS lookup on the IP address.
+ *
+ * @ip: The IP address to perform a reverse lookup on
+ *
+ * @return: success: Allocated string containing the hostname
+ * failure: NULL
+ */
+char *
+gf_rev_dns_lookup_cached (const char *ip, struct dnscache *dnscache)
+{
+ char *fqdn = NULL;
+ int ret = 0;
+ dict_t *cache = NULL;
+ data_t *entrydata = NULL;
+ struct dnscache_entry *dnsentry = NULL;
+ gf_boolean_t from_cache = _gf_false;
+
+ if (!dnscache)
+ goto out;
+
+ if (!dnscache->cache_dict) {
+ dnscache->cache_dict = dict_new ();
+ if (!dnscache->cache_dict) {
+ goto out;
+ }
+ }
+ cache = dnscache->cache_dict;
+
+ /* Quick cache lookup to see if we already hold it */
+ entrydata = dict_get (cache, (char *)ip);
+ if (entrydata) {
+ dnsentry = (struct dnscache_entry *)entrydata->data;
+ /* First check the TTL & timestamp */
+ if (time (NULL) - dnsentry->timestamp > dnscache->ttl) {
+ gf_dnscache_entry_deinit (dnsentry);
+ entrydata->data = NULL; /* Mark this as 'null' so
+ * dict_del () doesn't try free
+ * this after we've already
+ * freed it.
+ */
+
+ dict_del (cache, (char *)ip); /* Remove this entry */
+ } else {
+ /* Cache entry is valid, get the FQDN and return */
+ fqdn = dnsentry->fqdn;
+ from_cache = _gf_true; /* Mark this as from cache */
+ goto out;
+ }
+ }
+
+ /* Get the FQDN */
+ ret = gf_get_hostname_from_ip ((char *)ip, &fqdn);
+ if (ret != 0)
+ goto out;
+
+ if (!fqdn) {
+ gf_log_callingfn ("resolver", GF_LOG_CRITICAL,
+ "Allocation failed for the host address");
+ goto out;
+ }
+
+ from_cache = _gf_false;
+out:
+ /* Insert into the cache */
+ if (fqdn && !from_cache) {
+ struct dnscache_entry *entry = gf_dnscache_entry_init ();
+
+ if (!entry) {
+ goto out;
+ }
+ entry->fqdn = fqdn;
+ entry->ip = gf_strdup (ip);
+ if (!ip) {
+ gf_dnscache_entry_deinit (entry);
+ goto out;
+ }
+ entry->timestamp = time (NULL);
+
+ entrydata = bin_to_data (entry, sizeof (*entry));
+ dict_set (cache, (char *)ip, entrydata);
+ }
+ return fqdn;
+}
struct xldump {
int lineno;
@@ -4012,3 +4142,131 @@ _unmask_cancellation (void)
{
(void) pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, NULL);
}
+
+
+const char *
+fop_enum_to_pri_string (glusterfs_fop_t fop)
+{
+ switch (fop) {
+ case GF_FOP_OPEN:
+ case GF_FOP_STAT:
+ case GF_FOP_FSTAT:
+ case GF_FOP_LOOKUP:
+ case GF_FOP_ACCESS:
+ case GF_FOP_READLINK:
+ case GF_FOP_OPENDIR:
+ case GF_FOP_STATFS:
+ case GF_FOP_READDIR:
+ case GF_FOP_READDIRP:
+ return "HIGH";
+
+ case GF_FOP_CREATE:
+ case GF_FOP_FLUSH:
+ case GF_FOP_LK:
+ case GF_FOP_INODELK:
+ case GF_FOP_FINODELK:
+ case GF_FOP_ENTRYLK:
+ case GF_FOP_FENTRYLK:
+ case GF_FOP_UNLINK:
+ case GF_FOP_SETATTR:
+ case GF_FOP_FSETATTR:
+ case GF_FOP_MKNOD:
+ case GF_FOP_MKDIR:
+ case GF_FOP_RMDIR:
+ case GF_FOP_SYMLINK:
+ case GF_FOP_RENAME:
+ case GF_FOP_LINK:
+ case GF_FOP_SETXATTR:
+ case GF_FOP_GETXATTR:
+ case GF_FOP_FGETXATTR:
+ case GF_FOP_FSETXATTR:
+ case GF_FOP_REMOVEXATTR:
+ case GF_FOP_FREMOVEXATTR:
+ case GF_FOP_IPC:
+ return "NORMAL";
+
+ case GF_FOP_READ:
+ case GF_FOP_WRITE:
+ case GF_FOP_FSYNC:
+ case GF_FOP_TRUNCATE:
+ case GF_FOP_FTRUNCATE:
+ case GF_FOP_FSYNCDIR:
+ case GF_FOP_XATTROP:
+ case GF_FOP_FXATTROP:
+ case GF_FOP_RCHECKSUM:
+ case GF_FOP_ZEROFILL:
+ case GF_FOP_FALLOCATE:
+ return "LOW";
+
+ case GF_FOP_NULL:
+ case GF_FOP_FORGET:
+ case GF_FOP_RELEASE:
+ case GF_FOP_RELEASEDIR:
+ case GF_FOP_GETSPEC:
+ case GF_FOP_MAXVALUE:
+ case GF_FOP_DISCARD:
+ return "LEAST";
+ }
+ return "UNKNOWN";
+}
+
+const char *
+fop_enum_to_string (glusterfs_fop_t fop)
+{
+ static const char *const str_map[] = {
+ "NULL",
+ "STAT",
+ "READLINK",
+ "MKNOD",
+ "MKDIR",
+ "UNLINK",
+ "RMDIR",
+ "SYMLINK",
+ "RENAME",
+ "LINK",
+ "TRUNCATE",
+ "OPEN",
+ "READ",
+ "WRITE",
+ "STATFS",
+ "FLUSH",
+ "FSYNC",
+ "SETXATTR",
+ "GETXATTR",
+ "REMOVEXATTR",
+ "OPENDIR",
+ "FSYNCDIR",
+ "ACCESS",
+ "CREATE",
+ "FTRUNCATE",
+ "FSTAT",
+ "LK",
+ "LOOKUP",
+ "READDIR",
+ "INODELK",
+ "FINODELK",
+ "ENTRYLK",
+ "FENTRYLK",
+ "XATTROP",
+ "FXATTROP",
+ "FGETXATTR",
+ "FSETXATTR",
+ "RCHECKSUM",
+ "SETATTR",
+ "FSETATTR",
+ "READDIRP",
+ "FORGET",
+ "RELEASE",
+ "RELEASEDIR",
+ "GETSPEC",
+ "FREMOVEXATTR",
+ "FALLOCATE",
+ "DISCARD",
+ "ZEROFILL",
+ "IPC",
+ "MAXVALUE"};
+ if (fop <= GF_FOP_MAXVALUE)
+ return str_map[fop];
+
+ return "UNKNOWNFOP";
+}
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index 77a8cdd51c7..bf574fdabc7 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -159,6 +159,27 @@ typedef struct dht_changelog_rename_info {
typedef int (*gf_cmp) (void *, void *);
+struct _dict;
+
+struct dnscache {
+ struct _dict *cache_dict;
+ time_t ttl;
+};
+
+struct dnscache_entry {
+ char *ip;
+ char *fqdn;
+ time_t timestamp;
+};
+
+
+struct dnscache *gf_dnscache_init (time_t ttl);
+struct dnscache_entry *gf_dnscache_entry_init ();
+void gf_dnscache_entry_deinit (struct dnscache_entry *entry);
+char *gf_rev_dns_lookup_cached (const char *ip, struct dnscache *dnscache);
+
+char *gf_resolve_path_parent (const char *path);
+
void gf_global_variable_init(void);
int32_t gf_resolve_ip6 (const char *hostname, uint16_t port, int family,
diff --git a/libglusterfs/src/glusterfs.h b/libglusterfs/src/glusterfs.h
index 3bc76f6622a..4c7f9f517e3 100644
--- a/libglusterfs/src/glusterfs.h
+++ b/libglusterfs/src/glusterfs.h
@@ -258,6 +258,10 @@
/* NOTE: add members ONLY at the end (just before _MAXVALUE) */
+/*
+ * OTHER NOTE: fop_enum_to_str and fop_enum_to_pri_str (in common-utils.h) also
+ * contain lists of fops, so if you update this list UPDATE THOSE TOO.
+ */
typedef enum {
GF_FOP_NULL = 0,
GF_FOP_STAT,
@@ -312,6 +316,8 @@ typedef enum {
GF_FOP_MAXVALUE,
} glusterfs_fop_t;
+const char *fop_enum_to_pri_string (glusterfs_fop_t fop);
+const char *fop_enum_to_string (glusterfs_fop_t fop);
typedef enum {
GF_MGMT_NULL = 0,
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index 84949c61487..70c4ea770d5 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -128,6 +128,8 @@ enum gf_common_mem_types_ {
gf_common_mt_ereg,
gf_common_mt_wr,
gf_common_mt_rdma_arena_mr,
+ gf_common_mt_dnscache = 115,
+ gf_common_mt_dnscache_entry = 116,
gf_common_mt_parser_t,
gf_common_quota_meta_t,
/*related to gfdb library*/
diff --git a/libglusterfs/src/stack.h b/libglusterfs/src/stack.h
index 5c0655f2ead..43d943c62b6 100644
--- a/libglusterfs/src/stack.h
+++ b/libglusterfs/src/stack.h
@@ -96,6 +96,7 @@ struct _call_stack_t {
uid_t uid;
gid_t gid;
pid_t pid;
+ char identifier[UNIX_PATH_MAX];
uint16_t ngrps;
uint32_t groups_small[SMALL_GROUP_COUNT];
uint32_t *groups_large;
diff --git a/tests/basic/fop-sampling.t b/tests/basic/fop-sampling.t
new file mode 100644
index 00000000000..cea8aa737c0
--- /dev/null
+++ b/tests/basic/fop-sampling.t
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+
+. $(dirname $0)/../include.rc
+. $(dirname $0)/../volume.rc
+
+SAMPLE_FILE="$(gluster --print-logdir)/samples/glusterfs_${V0}.samp"
+
+function print_cnt() {
+ local FOP_TYPE=$1
+ local FOP_CNT=$(grep ,${FOP_TYPE} ${SAMPLE_FILE} | wc -l)
+ echo $FOP_CNT
+}
+
+# Verify we got non-zero counts for stats/lookup/readdir
+check_samples() {
+ STAT_CNT=$(print_cnt STAT)
+ if [ "$STAT_CNT" -le "0" ]; then
+ echo "STAT count is zero"
+ return
+ fi
+
+ LOOKUP_CNT=$(print_cnt LOOKUP)
+ if [ "$LOOKUP_CNT" -le "0" ]; then
+ echo "LOOKUP count is zero"
+ return
+ fi
+
+ READDIR_CNT=$(print_cnt READDIR)
+ if [ "$READDIR_CNT" -le "0" ]; then
+ echo "READDIR count is zero"
+ return
+ fi
+
+ echo "OK"
+}
+
+cleanup;
+TEST glusterd
+TEST pidof glusterd
+TEST $CLI volume create $V0 replica 3 $H0:$B0/${V0}{0,1,2}
+TEST $CLI volume set $V0 nfs.disable off
+TEST $CLI volume set $V0 diagnostics.latency-measurement on
+TEST $CLI volume set $V0 diagnostics.count-fop-hits on
+TEST $CLI volume set $V0 diagnostics.stats-dump-interval 2
+TEST $CLI volume set $V0 diagnostics.fop-sample-buf-size 65535
+TEST $CLI volume set $V0 diagnostics.fop-sample-interval 1
+TEST $CLI volume set $V0 diagnostics.stats-dnscache-ttl-sec 3600
+
+TEST $CLI volume start $V0
+TEST glusterfs --volfile-id=/$V0 --volfile-server=$H0 $M0 --attribute-timeout=0 --entry-timeout=0
+
+for i in {1..5}
+do
+ dd if=/dev/zero of=${M0}/testfile$i bs=4k count=1
+done
+
+TEST ls -l $M0
+EXPECT_WITHIN 6 "OK" check_samples
+
+cleanup
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index d7a1055a571..9dde9373264 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -20,6 +20,8 @@ enum gf_io_stats_mem_types_ {
gf_io_stats_mt_ios_fd,
gf_io_stats_mt_ios_stat,
gf_io_stats_mt_ios_stat_list,
+ gf_io_stats_mt_ios_sample_buf,
+ gf_io_stats_mt_ios_sample,
gf_io_stats_mt_end
};
#endif
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 3157a65dae1..e73cae0a928 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -34,8 +34,12 @@
#include "logging.h"
#include "cli1-xdr.h"
#include "statedump.h"
+#include <pwd.h>
+#include <grp.h>
#define MAX_LIST_MEMBERS 100
+#define DEFAULT_PWD_BUF_SZ 16384
+#define DEFAULT_GRP_BUF_SZ 16384
typedef enum {
IOS_STATS_TYPE_NONE,
@@ -82,6 +86,25 @@ struct ios_stat_head {
struct ios_stat_list *iosstats;
};
+typedef struct _ios_sample_t {
+ uid_t uid;
+ gid_t gid;
+ char identifier[UNIX_PATH_MAX];
+ glusterfs_fop_t fop_type;
+ struct timeval timestamp;
+ double elapsed;
+} ios_sample_t;
+
+
+typedef struct _ios_sample_buf_t {
+ uint64_t pos; /* Position in write buffer */
+ uint64_t size; /* Size of ring buffer */
+ uint64_t collected; /* Number of samples we've collected */
+ uint64_t observed; /* Number of FOPs we've observed */
+ ios_sample_t *ios_samples; /* Our list of samples */
+} ios_sample_buf_t;
+
+
struct ios_lat {
double min;
double max;
@@ -102,7 +125,6 @@ struct ios_global_stats {
struct timeval max_openfd_time;
};
-
struct ios_conf {
gf_lock_t lock;
struct ios_global_stats cumulative;
@@ -116,6 +138,12 @@ struct ios_conf {
int32_t ios_dump_interval;
pthread_t dump_thread;
gf_boolean_t dump_thread_should_die;
+ gf_lock_t ios_sampling_lock;
+ int32_t ios_sample_interval;
+ int32_t ios_sample_buf_size;
+ ios_sample_buf_t *ios_sample_buf;
+ struct dnscache *dnscache;
+ int32_t ios_dnscache_ttl_sec;
};
@@ -133,7 +161,8 @@ typedef enum {
IOS_DUMP_TYPE_FILE = 1,
IOS_DUMP_TYPE_DICT = 2,
IOS_DUMP_TYPE_JSON_FILE = 3,
- IOS_DUMP_TYPE_MAX = 4
+ IOS_DUMP_TYPE_SAMPLES = 4,
+ IOS_DUMP_TYPE_MAX = 5
} ios_dump_type_t;
struct ios_dump_args {
@@ -163,6 +192,7 @@ is_fop_latency_started (call_frame_t *frame)
}
#ifdef GF_LINUX_HOST_OS
+const char *_IOS_SAMP_DIR = "/var/log/glusterfs/samples";
#define _IOS_DUMP_DIR DATADIR "/lib/glusterd/stats"
#else
#define _IOS_DUMP_DIR DATADIR "/db/glusterd/stats"
@@ -429,6 +459,72 @@ ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
}
+/*
+ * So why goto all this trouble? Why not just queue up some samples in
+ * a big list and malloc away? Well malloc is expensive relative
+ * to what we are measuring, so cannot have any malloc's (or worse
+ * callocs) in our measurement code paths. Instead, we are going to
+ * pre-allocate a circular buffer and collect a maximum number of samples.
+ * Prior to dumping them all we'll create a new buffer and swap the
+ * old buffer with the new, and then proceed to dump the statistics
+ * in our dump thread.
+ *
+ */
+ios_sample_buf_t *
+ios_create_sample_buf (size_t buf_size)
+{
+ ios_sample_buf_t *ios_sample_buf = NULL;
+ ios_sample_t *ios_samples = NULL;
+
+ ios_sample_buf = GF_CALLOC (1,
+ sizeof (*ios_sample_buf),
+ gf_io_stats_mt_ios_sample_buf);
+ if (!ios_sample_buf)
+ goto err;
+
+ ios_samples = GF_CALLOC (buf_size,
+ sizeof (*ios_samples),
+ gf_io_stats_mt_ios_sample);
+
+ if (!ios_samples)
+ goto err;
+
+ ios_sample_buf->ios_samples = ios_samples;
+ ios_sample_buf->size = buf_size;
+ ios_sample_buf->pos = 0;
+ ios_sample_buf->observed = 0;
+ ios_sample_buf->collected = 0;
+
+ return ios_sample_buf;
+err:
+ GF_FREE (ios_sample_buf);
+ return NULL;
+}
+
+void
+ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)
+{
+ GF_FREE (ios_sample_buf->ios_samples);
+ GF_FREE (ios_sample_buf);
+}
+
+static int
+ios_init_sample_buf (struct ios_conf *conf)
+{
+ int32_t ret = -1;
+
+ GF_ASSERT (conf);
+ LOCK (&conf->lock);
+ conf->ios_sample_buf = ios_create_sample_buf (
+ conf->ios_sample_buf_size);
+ if (!conf->ios_sample_buf)
+ goto out;
+ ret = 0;
+out:
+ UNLOCK (&conf->lock);
+ return ret;
+}
+
int
ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
struct ios_stat *iosstat)
@@ -815,6 +911,230 @@ out:
return ret;
}
+char *
+_resolve_username (xlator_t *this, uid_t uid)
+{
+ struct passwd pwd;
+ struct passwd *pwd_result = NULL;
+ size_t pwd_buf_len;
+ char *pwd_buf = NULL;
+ char *ret = NULL;
+
+ /* Prepare our buffer for the uid->username translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+ pwd_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+#else
+ pwd_buf_len = -1;
+#endif
+ if (pwd_buf_len == -1) {
+ pwd_buf_len = DEFAULT_PWD_BUF_SZ; /* per the man page */
+ }
+
+ pwd_buf = alloca (pwd_buf_len);
+ if (!pwd_buf)
+ goto err;
+
+ getpwuid_r (uid, &pwd, pwd_buf, pwd_buf_len,
+ &pwd_result);
+ if (!pwd_result)
+ goto err;
+
+ ret = gf_strdup (pwd.pw_name);
+ if (ret)
+ return ret;
+ else
+ gf_log (this->name, GF_LOG_ERROR,
+ "gf_strdup failed, failing username "
+ "resolution.");
+err:
+ return ret;
+}
+
+char *
+_resolve_group_name (xlator_t *this, gid_t gid)
+{
+ struct group grp;
+ struct group *grp_result = NULL;
+ size_t grp_buf_len;
+ char *grp_buf = NULL;
+ char *ret = NULL;
+
+ /* Prepare our buffer for the gid->group name translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+ grp_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+#else
+ grp_buf_len = -1;
+#endif
+ if (grp_buf_len == -1) {
+ grp_buf_len = DEFAULT_GRP_BUF_SZ; /* per the man page */
+ }
+
+ grp_buf = alloca (grp_buf_len);
+ if (!grp_buf) {
+ goto err;
+ }
+
+ getgrgid_r (gid, &grp, grp_buf, grp_buf_len,
+ &grp_result);
+ if (!grp_result)
+ goto err;
+
+ ret = gf_strdup (grp.gr_name);
+ if (ret)
+ return ret;
+ else
+ gf_log (this->name, GF_LOG_ERROR,
+ "gf_strdup failed, failing username "
+ "resolution.");
+err:
+ return ret;
+}
+
+
+/*
+ * This function writes out a latency sample to a given file descriptor
+ * and beautifies the output in the process.
+ */
+void
+_io_stats_write_latency_sample (xlator_t *this, ios_sample_t *sample,
+ FILE *logfp)
+{
+ double epoch_time = 0.00;
+ char *xlator_name = NULL;
+ char *instance_name = NULL;
+ char *hostname = NULL;
+ char *identifier = NULL;
+ char *port = NULL;
+ char *port_pos = NULL;
+ char *group_name = NULL;
+ char *username = NULL;
+ struct ios_conf *conf = NULL;
+
+ conf = this->private;
+
+ epoch_time = (sample->timestamp).tv_sec +
+ ((sample->timestamp).tv_usec / 1000000.0);
+
+ if (!sample->identifier || (strlen (sample->identifier) == 0)) {
+ hostname = "Unknown";
+ port = "Unknown";
+ } else {
+ identifier = strdupa (sample->identifier);
+ port_pos = strrchr (identifier, ':');
+ if (!port_pos || strlen(port_pos) < 2)
+ goto err;
+ port = strdupa (port_pos + 1);
+ if (!port)
+ goto err;
+ *port_pos = '\0';
+ hostname = gf_rev_dns_lookup_cached (identifier,
+ conf->dnscache);
+ if (!hostname)
+ hostname = "Unknown";
+ }
+
+ xlator_name = this->name;
+ if (!xlator_name || strlen (xlator_name) == 0)
+ xlator_name = "Unknown";
+
+ instance_name = this->instance_name;
+ if (!instance_name || strlen (instance_name) == 0)
+ instance_name = "N/A";
+
+ /* Resolve the UID to a string username */
+ username = _resolve_username (this, sample->uid);
+ if (!username) {
+ username = GF_MALLOC (30, gf_common_mt_char);
+ sprintf (username, "%d", (int32_t)sample->uid);
+ }
+
+ /* Resolve the GID to a string group name */
+ group_name = _resolve_group_name (this, sample->gid);
+ if (!group_name) {
+ group_name = GF_MALLOC (30, gf_common_mt_char);
+ sprintf (group_name, "%d", (int32_t)sample->gid);
+ }
+
+ ios_log (this, logfp,
+ "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s",
+ epoch_time, fop_enum_to_pri_string (sample->fop_type),
+ fop_enum_to_string (sample->fop_type),
+ sample->elapsed, xlator_name, instance_name, username,
+ group_name, hostname, port);
+ goto out;
+err:
+ gf_log (this->name, GF_LOG_ERROR,
+ "Error parsing socket identifier");
+out:
+ GF_FREE (group_name);
+ GF_FREE (username);
+}
+
+/*
+ * Takes our current sample buffer in conf->io_sample_buf, and saves
+ * a reference to this, init's a new buffer, and then dumps out the
+ * contents of the saved reference.
+ */
+int
+io_stats_dump_latency_samples_logfp (xlator_t *this, FILE *logfp)
+{
+ uint64_t i = 0;
+ struct ios_conf *conf = NULL;
+ ios_sample_buf_t *sample_buf = NULL;
+ int ret = 1; /* Default to error */
+
+ conf = this->private;
+
+ /* Save pointer to old buffer; the CS equivalent of
+ * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
+ * though ours will end better I hope!
+ */
+ sample_buf = conf->ios_sample_buf;
+ if (!sample_buf) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Sampling buffer is null, bailing!");
+ goto out;
+ }
+
+ /* Empty case, nothing to do, exit. */
+ if (sample_buf->collected == 0) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "No samples, dump not required.");
+ ret = 0;
+ goto out;
+ }
+
+ /* Init a new buffer, so we are free to work on the one we saved a
+ * reference to above.
+ */
+ if (ios_init_sample_buf (conf) != 0) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Failed to init new sampling buffer, out of memory?");
+ goto out;
+ }
+
+ /* Wrap-around case, dump from pos to sample_buf->size -1
+ * and then from 0 to sample_buf->pos (covered off by
+ * "simple case")
+ */
+ if (sample_buf->collected > sample_buf->pos + 1) {
+ for (i = sample_buf->pos; i < sample_buf->size; i++) {
+ _io_stats_write_latency_sample (this,
+ &(sample_buf->ios_samples[i]), logfp);
+ }
+ }
+
+ /* Simple case: Dump from 0 to sample_buf->pos */
+ for (i = 0; i < sample_buf->pos; i++) {
+ _io_stats_write_latency_sample (this,
+ &(sample_buf->ios_samples[i]), logfp);
+ }
+ ios_destroy_sample_buf (sample_buf);
+
+out:
+ return ret;
+}
+
int
io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
struct timeval *now, int interval, FILE* logfp)
@@ -1284,6 +1604,48 @@ io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
return 0;
}
+void collect_ios_latency_sample (struct ios_conf *conf,
+ glusterfs_fop_t fop_type, double elapsed,
+ call_frame_t *frame)
+{
+ ios_sample_buf_t *ios_sample_buf = NULL;
+ ios_sample_t *ios_sample = NULL;
+ struct timeval *timestamp = NULL;
+ call_stack_t *root = NULL;
+
+
+ ios_sample_buf = conf->ios_sample_buf;
+ LOCK (&conf->ios_sampling_lock);
+ if (conf->ios_sample_interval == 0 ||
+ ios_sample_buf->observed % conf->ios_sample_interval != 0)
+ goto out;
+
+ timestamp = &frame->begin;
+ root = frame->root;
+
+ ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
+ ios_sample->elapsed = elapsed;
+ ios_sample->fop_type = fop_type;
+ ios_sample->uid = root->uid;
+ ios_sample->gid = root->gid;
+ (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
+ (ios_sample->timestamp).tv_usec = timestamp->tv_usec;
+ memcpy (&ios_sample->identifier, &root->identifier,
+ sizeof (root->identifier));
+
+ /* We've reached the end of the circular buffer, start from the
+ * beginning. */
+ if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
+ ios_sample_buf->pos = 0;
+ else
+ ios_sample_buf->pos++;
+ ios_sample_buf->collected++;
+out:
+ ios_sample_buf->observed++;
+ UNLOCK (&conf->ios_sampling_lock);
+ return;
+}
+
static void
update_ios_latency_stats (struct ios_global_stats *stats, double elapsed,
glusterfs_fop_t op)
@@ -1321,6 +1683,7 @@ update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
update_ios_latency_stats (&conf->cumulative, elapsed, op);
update_ios_latency_stats (&conf->incremental, elapsed, op);
+ collect_ios_latency_sample (conf, op, elapsed, frame);
return 0;
}
@@ -1660,7 +2023,7 @@ io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
if (iosstat) {
BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
- iosstat = NULL;
+ iosstat = NULL;
}
STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
@@ -2512,14 +2875,18 @@ _ios_destroy_dump_thread (struct ios_conf *conf) {
void *
_ios_dump_thread (xlator_t *this) {
struct ios_conf *conf = NULL;
- FILE *logfp = NULL;
+ FILE *stats_logfp = NULL;
+ FILE *samples_logfp = NULL;
struct ios_dump_args args = {0};
int i;
- int bytes_written = 0;
- char filename[PATH_MAX];
+ int stats_bytes_written = 0;
+ int samples_bytes_written = 0;
+ char stats_filename[PATH_MAX];
+ char samples_filename[PATH_MAX];
char *xlator_name;
char *instance_name;
- gf_boolean_t log_fopen_failure = _gf_true;
+ gf_boolean_t log_stats_fopen_failure = _gf_true;
+ gf_boolean_t log_samples_fopen_failure = _gf_true;
int old_cancel_type;
conf = this->private;
@@ -2546,17 +2913,34 @@ _ios_dump_thread (xlator_t *this) {
goto out;
}
}
+ if (mkdir (_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+ if (errno != EEXIST) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not create stats-sample directory %s",
+ _IOS_SAMP_DIR);
+ goto out;
+ }
+ }
if (instance_name) {
- bytes_written = snprintf (filename, PATH_MAX,
+ stats_bytes_written = snprintf (stats_filename, PATH_MAX,
"%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
__progname, xlator_name, instance_name);
+ samples_bytes_written = snprintf (samples_filename, PATH_MAX,
+ "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
+ __progname, xlator_name, instance_name);
} else {
- bytes_written = snprintf (filename, PATH_MAX, "%s/%s_%s.dump",
- _IOS_DUMP_DIR, __progname, xlator_name);
- }
- if (bytes_written >= PATH_MAX) {
+ stats_bytes_written = snprintf (stats_filename, PATH_MAX,
+ "%s/%s_%s.dump", _IOS_DUMP_DIR, __progname,
+ xlator_name);
+ samples_bytes_written = snprintf (samples_filename, PATH_MAX,
+ "%s/%s_%s.samp", _IOS_SAMP_DIR, __progname,
+ xlator_name);
+ }
+ if ((stats_bytes_written >= PATH_MAX) ||
+ (samples_bytes_written >= PATH_MAX)) {
gf_log (this->name, GF_LOG_ERROR,
- "Invalid path for IO Stats dump: %s", filename);
+ "Invalid path for stats dump (%s) and/or latency "
+ "samples (%s)", stats_filename, samples_filename);
goto out;
}
while (1) {
@@ -2572,22 +2956,32 @@ _ios_dump_thread (xlator_t *this) {
* just hold it open and rewind/truncate on each iteration.
* Leaving it alone for now.
*/
- logfp = fopen (filename, "w+");
- if (!logfp) {
- if (log_fopen_failure) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not open stats-dump file %s",
- filename);
- log_fopen_failure = _gf_false;
- }
- continue;
+ stats_logfp = fopen (stats_filename, "w+");
+ if (stats_logfp) {
+ (void) ios_dump_args_init (&args,
+ IOS_DUMP_TYPE_JSON_FILE,
+ stats_logfp);
+ io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
+ fclose (stats_logfp);
+ log_stats_fopen_failure = _gf_true;
+ } else if (log_stats_fopen_failure) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not open stats-dump file %s (%s)",
+ stats_filename, strerror(errno));
+ log_stats_fopen_failure = _gf_false;
+ }
+ samples_logfp = fopen (samples_filename, "w+");
+ if (samples_logfp) {
+ io_stats_dump_latency_samples_logfp (this,
+ samples_logfp);
+ fclose (samples_logfp);
+ log_samples_fopen_failure = _gf_true;
+ } else if (log_samples_fopen_failure) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not open samples-dump file %s (%s)",
+ samples_filename, strerror(errno));
+ log_samples_fopen_failure = _gf_false;
}
- (void) ios_dump_args_init (
- &args, IOS_DUMP_TYPE_JSON_FILE,
- logfp);
- io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
- fclose (logfp);
- log_fopen_failure = _gf_true;
}
out:
gf_log (this->name, GF_LOG_INFO, "IO stats dump thread terminated");
@@ -3116,6 +3510,10 @@ reconfigure (xlator_t *this, dict_t *options)
(void *) &_ios_dump_thread, this);
}
+ GF_OPTION_RECONF ("ios-sample-interval", conf->ios_sample_interval,
+ options, int32, out);
+ GF_OPTION_RECONF ("ios-sample-buf-size", conf->ios_sample_buf_size,
+ options, int32, out);
GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
if (sys_log_str) {
sys_log_level = glusterd_check_log_level (sys_log_str);
@@ -3229,6 +3627,7 @@ init (xlator_t *this)
* in case of error paths.
*/
LOCK_INIT (&conf->lock);
+ LOCK_INIT (&conf->ios_sampling_lock);
gettimeofday (&conf->cumulative.started_at, NULL);
gettimeofday (&conf->incremental.started_at, NULL);
@@ -3247,6 +3646,22 @@ init (xlator_t *this)
GF_OPTION_INIT ("ios-dump-interval", conf->ios_dump_interval,
int32, out);
+ GF_OPTION_INIT ("ios-sample-interval", conf->ios_sample_interval,
+ int32, out);
+
+ GF_OPTION_INIT ("ios-sample-buf-size", conf->ios_sample_buf_size,
+ int32, out);
+
+ if (ios_init_sample_buf (conf) != 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Out of memory.");
+ return -1;
+ }
+
+ GF_OPTION_INIT ("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec,
+ int32, out);
+ conf->dnscache = gf_dnscache_init (conf->ios_dnscache_ttl_sec);
+
GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
if (sys_log_str) {
sys_log_level = glusterd_check_log_level (sys_log_str);
@@ -3497,6 +3912,29 @@ struct volume_options options[] = {
.description = "Interval (in seconds) at which to auto-dump "
"statistics. Zero disables automatic dumping."
},
+ { .key = { "ios-sample-interval" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 0,
+ .max = 65535,
+ .default_value = "0",
+ .description = "Interval in which we want to collect FOP latency "
+ "samples. 2 means collect a sample every 2nd FOP."
+ },
+ { .key = { "ios-sample-buf-size" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 1024,
+ .max = 1024*1024,
+ .default_value = "65535",
+ .description = "The maximum size of our FOP sampling ring buffer."
+ },
+ { .key = { "ios-dnscache-ttl-sec" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 1,
+ .max = 3600 * 72,
+ .default_value = "86400",
+ .description = "The interval after wish a cached DNS entry will be "
+ "re-validated. Default: 24 hrs"
+ },
{ .key = { "latency-measurement" },
.type = GF_OPTION_TYPE_BOOL,
.default_value = "off",
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 4ea7d3baa8a..ccf4b380ffb 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -849,6 +849,21 @@ struct volopt_map_entry glusterd_volopt_map[] = {
.option = "ios-dump-interval",
.op_version = 1
},
+ { .key = "diagnostics.fop-sample-interval",
+ .voltype = "debug/io-stats",
+ .option = "ios-sample-interval",
+ .op_version = 1
+ },
+ { .key = "diagnostics.fop-sample-buf-size",
+ .voltype = "debug/io-stats",
+ .option = "ios-sample-buf-size",
+ .op_version = 1
+ },
+ { .key = "diagnostics.stats-dnscache-ttl-sec",
+ .voltype = "debug/io-stats",
+ .option = "ios-dnscache-ttl-sec",
+ .op_version = 1
+ },
/* IO-cache xlator options */
{ .key = "performance.cache-max-file-size",
diff --git a/xlators/nfs/server/src/nfs-fops.c b/xlators/nfs/server/src/nfs-fops.c
index 819ae98b081..52fdd9796c3 100644
--- a/xlators/nfs/server/src/nfs-fops.c
+++ b/xlators/nfs/server/src/nfs-fops.c
@@ -219,6 +219,7 @@ nfs_create_frame (xlator_t *xl, nfs_user_t *nfu)
frame->root->pid = NFS_PID;
frame->root->uid = nfu->uid;
frame->root->gid = nfu->gids[NFS_PRIMGID_IDX];
+ memcpy (&frame->root->identifier, &nfu->identifier, UNIX_PATH_MAX);
frame->root->lk_owner = nfu->lk_owner;
if (nfu->ngrps != 1) {
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 4dda35c49ef..116854b9109 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -639,8 +639,8 @@ nfs_user_root_create (nfs_user_t *newnfu)
int
-nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid, gid_t *auxgids,
- int auxcount)
+nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid,
+ rpc_transport_t *trans, gid_t *auxgids, int auxcount)
{
int x = 1;
int y = 0;
@@ -655,6 +655,10 @@ nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid, gid_t *auxgids,
newnfu->uid = uid;
newnfu->gids[0] = gid;
newnfu->ngrps = 1;
+ if (trans) {
+ memcpy (&newnfu->identifier, trans->peerinfo.identifier,
+ UNIX_PATH_MAX);
+ }
gf_msg_trace (GF_NFS, 0, "uid: %d, gid %d, gids: %d", uid, gid,
auxcount);
@@ -683,7 +687,9 @@ nfs_request_user_init (nfs_user_t *nfu, rpcsvc_request_t *req)
gidarr = rpcsvc_auth_unix_auxgids (req, &gids);
nfs_user_create (nfu, rpcsvc_request_uid (req),
- rpcsvc_request_gid (req), gidarr, gids);
+ rpcsvc_request_gid (req),
+ rpcsvc_request_transport (req),
+ gidarr, gids);
return;
}
@@ -699,7 +705,8 @@ nfs_request_primary_user_init (nfs_user_t *nfu, rpcsvc_request_t *req,
return;
gidarr = rpcsvc_auth_unix_auxgids (req, &gids);
- nfs_user_create (nfu, uid, gid, gidarr, gids);
+ nfs_user_create (nfu, uid, gid, rpcsvc_request_transport (req),
+ gidarr, gids);
return;
}
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index 107140bc720..82df163d494 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -125,14 +125,15 @@ typedef struct nfs_user_info {
gid_t gids[NFS_NGROUPS];
int ngrps;
gf_lkowner_t lk_owner;
+ char identifier[UNIX_PATH_MAX]; /* ip of user */
} nfs_user_t;
extern int
nfs_user_root_create (nfs_user_t *newnfu);
extern int
-nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid, gid_t *auxgids,
- int auxcount);
+nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid,
+ rpc_transport_t *trans, gid_t *auxgids, int auxcount);
extern void
nfs_request_user_init (nfs_user_t *nfu, rpcsvc_request_t *req);
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index bcd1fed8b84..b369e835b24 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -423,6 +423,7 @@ get_frame_from_request (rpcsvc_request_t *req)
server_conf_t *priv = NULL;
clienttable_t *clienttable = NULL;
unsigned int i = 0;
+ rpc_transport_t *trans = NULL;
GF_VALIDATE_OR_GOTO ("server", req, out);
@@ -500,6 +501,12 @@ get_frame_from_request (rpcsvc_request_t *req)
server_resolve_groups (frame, req);
else
server_decode_groups (frame, req);
+ trans = req->trans;
+ if (trans) {
+ memcpy (&frame->root->identifier, trans->peerinfo.identifier,
+ sizeof (trans->peerinfo.identifier));
+ }
+
frame->local = req;
out: