summaryrefslogtreecommitdiffstats
path: root/xlators/debug
diff options
context:
space:
mode:
authorRichard Wareing <rwareing@fb.com>2015-06-23 17:03:11 -0700
committerVijay Bellur <vbellur@redhat.com>2015-11-01 09:14:34 -0800
commitd3e496cbcd35b9d9b840e328ae109c44f59083ce (patch)
tree25eedba238f9168cd11015885973d5a960c1596e /xlators/debug
parent846697d91ed56ca3d76f2a78e87e7675f127f21d (diff)
debug/io-stats: Add FOP sampling feature
Summary: - Using sampling feature you can record details about every Nth FOP. The fields in each sample are: FOP type, hostname, uid, gid, FOP priority, port and time taken (latency) to fufill the request. - Implemented using a ring buffer which is not (m/c) allocated in the IO path, this should make the sampling process pretty cheap. - DNS resolution done @ dump time not @ sample time for performance w/ cache - Metrics can be used for both diagnostics, traffic/IO profiling as well as P95/P99 calculations - To control this feature there are two new volume options: diagnostics.fop-sample-interval - The sampling interval, e.g. 1 means sample every FOP, 100 means sample every 100th FOP diagnostics.fop-sample-buf-size - The size (in bytes) of the ring buffer used to store the samples. In the even more samples are collected in the stats dump interval than can be held in this buffer, the oldest samples shall be discarded. Samples are stored in the log directory under /var/log/glusterfs/samples. - Uses DNS cache written by sshreyas@fb.com (Thank-you!), the DNS cache TTL is controlled by the diagnostics.stats-dnscache-ttl-sec option and defaults to 24hrs. Test Plan: - Valgrind'd to ensure it's leak free - Run prove test(s) - Shadow testing on 100+ brick cluster Change-Id: I9ee14c2fa18486b7efb38e59f70687249d3f96d8 BUG: 1271310 Signed-off-by: Jeff Darcy <jdarcy@redhat.com> Reviewed-on: http://review.gluster.org/12210 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com>
Diffstat (limited to 'xlators/debug')
-rw-r--r--xlators/debug/io-stats/src/io-stats-mem-types.h2
-rw-r--r--xlators/debug/io-stats/src/io-stats.c494
2 files changed, 468 insertions, 28 deletions
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index d7a1055a571..9dde9373264 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -20,6 +20,8 @@ enum gf_io_stats_mem_types_ {
gf_io_stats_mt_ios_fd,
gf_io_stats_mt_ios_stat,
gf_io_stats_mt_ios_stat_list,
+ gf_io_stats_mt_ios_sample_buf,
+ gf_io_stats_mt_ios_sample,
gf_io_stats_mt_end
};
#endif
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index 3157a65dae1..e73cae0a928 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -34,8 +34,12 @@
#include "logging.h"
#include "cli1-xdr.h"
#include "statedump.h"
+#include <pwd.h>
+#include <grp.h>
#define MAX_LIST_MEMBERS 100
+#define DEFAULT_PWD_BUF_SZ 16384
+#define DEFAULT_GRP_BUF_SZ 16384
typedef enum {
IOS_STATS_TYPE_NONE,
@@ -82,6 +86,25 @@ struct ios_stat_head {
struct ios_stat_list *iosstats;
};
+typedef struct _ios_sample_t {
+ uid_t uid;
+ gid_t gid;
+ char identifier[UNIX_PATH_MAX];
+ glusterfs_fop_t fop_type;
+ struct timeval timestamp;
+ double elapsed;
+} ios_sample_t;
+
+
+typedef struct _ios_sample_buf_t {
+ uint64_t pos; /* Position in write buffer */
+ uint64_t size; /* Size of ring buffer */
+ uint64_t collected; /* Number of samples we've collected */
+ uint64_t observed; /* Number of FOPs we've observed */
+ ios_sample_t *ios_samples; /* Our list of samples */
+} ios_sample_buf_t;
+
+
struct ios_lat {
double min;
double max;
@@ -102,7 +125,6 @@ struct ios_global_stats {
struct timeval max_openfd_time;
};
-
struct ios_conf {
gf_lock_t lock;
struct ios_global_stats cumulative;
@@ -116,6 +138,12 @@ struct ios_conf {
int32_t ios_dump_interval;
pthread_t dump_thread;
gf_boolean_t dump_thread_should_die;
+ gf_lock_t ios_sampling_lock;
+ int32_t ios_sample_interval;
+ int32_t ios_sample_buf_size;
+ ios_sample_buf_t *ios_sample_buf;
+ struct dnscache *dnscache;
+ int32_t ios_dnscache_ttl_sec;
};
@@ -133,7 +161,8 @@ typedef enum {
IOS_DUMP_TYPE_FILE = 1,
IOS_DUMP_TYPE_DICT = 2,
IOS_DUMP_TYPE_JSON_FILE = 3,
- IOS_DUMP_TYPE_MAX = 4
+ IOS_DUMP_TYPE_SAMPLES = 4,
+ IOS_DUMP_TYPE_MAX = 5
} ios_dump_type_t;
struct ios_dump_args {
@@ -163,6 +192,7 @@ is_fop_latency_started (call_frame_t *frame)
}
#ifdef GF_LINUX_HOST_OS
+const char *_IOS_SAMP_DIR = "/var/log/glusterfs/samples";
#define _IOS_DUMP_DIR DATADIR "/lib/glusterd/stats"
#else
#define _IOS_DUMP_DIR DATADIR "/db/glusterd/stats"
@@ -429,6 +459,72 @@ ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
}
+/*
+ * So why goto all this trouble? Why not just queue up some samples in
+ * a big list and malloc away? Well malloc is expensive relative
+ * to what we are measuring, so cannot have any malloc's (or worse
+ * callocs) in our measurement code paths. Instead, we are going to
+ * pre-allocate a circular buffer and collect a maximum number of samples.
+ * Prior to dumping them all we'll create a new buffer and swap the
+ * old buffer with the new, and then proceed to dump the statistics
+ * in our dump thread.
+ *
+ */
+ios_sample_buf_t *
+ios_create_sample_buf (size_t buf_size)
+{
+ ios_sample_buf_t *ios_sample_buf = NULL;
+ ios_sample_t *ios_samples = NULL;
+
+ ios_sample_buf = GF_CALLOC (1,
+ sizeof (*ios_sample_buf),
+ gf_io_stats_mt_ios_sample_buf);
+ if (!ios_sample_buf)
+ goto err;
+
+ ios_samples = GF_CALLOC (buf_size,
+ sizeof (*ios_samples),
+ gf_io_stats_mt_ios_sample);
+
+ if (!ios_samples)
+ goto err;
+
+ ios_sample_buf->ios_samples = ios_samples;
+ ios_sample_buf->size = buf_size;
+ ios_sample_buf->pos = 0;
+ ios_sample_buf->observed = 0;
+ ios_sample_buf->collected = 0;
+
+ return ios_sample_buf;
+err:
+ GF_FREE (ios_sample_buf);
+ return NULL;
+}
+
+void
+ios_destroy_sample_buf (ios_sample_buf_t *ios_sample_buf)
+{
+ GF_FREE (ios_sample_buf->ios_samples);
+ GF_FREE (ios_sample_buf);
+}
+
+static int
+ios_init_sample_buf (struct ios_conf *conf)
+{
+ int32_t ret = -1;
+
+ GF_ASSERT (conf);
+ LOCK (&conf->lock);
+ conf->ios_sample_buf = ios_create_sample_buf (
+ conf->ios_sample_buf_size);
+ if (!conf->ios_sample_buf)
+ goto out;
+ ret = 0;
+out:
+ UNLOCK (&conf->lock);
+ return ret;
+}
+
int
ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
struct ios_stat *iosstat)
@@ -815,6 +911,230 @@ out:
return ret;
}
+char *
+_resolve_username (xlator_t *this, uid_t uid)
+{
+ struct passwd pwd;
+ struct passwd *pwd_result = NULL;
+ size_t pwd_buf_len;
+ char *pwd_buf = NULL;
+ char *ret = NULL;
+
+ /* Prepare our buffer for the uid->username translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+ pwd_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+#else
+ pwd_buf_len = -1;
+#endif
+ if (pwd_buf_len == -1) {
+ pwd_buf_len = DEFAULT_PWD_BUF_SZ; /* per the man page */
+ }
+
+ pwd_buf = alloca (pwd_buf_len);
+ if (!pwd_buf)
+ goto err;
+
+ getpwuid_r (uid, &pwd, pwd_buf, pwd_buf_len,
+ &pwd_result);
+ if (!pwd_result)
+ goto err;
+
+ ret = gf_strdup (pwd.pw_name);
+ if (ret)
+ return ret;
+ else
+ gf_log (this->name, GF_LOG_ERROR,
+ "gf_strdup failed, failing username "
+ "resolution.");
+err:
+ return ret;
+}
+
+char *
+_resolve_group_name (xlator_t *this, gid_t gid)
+{
+ struct group grp;
+ struct group *grp_result = NULL;
+ size_t grp_buf_len;
+ char *grp_buf = NULL;
+ char *ret = NULL;
+
+ /* Prepare our buffer for the gid->group name translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+ grp_buf_len = sysconf (_SC_GETGR_R_SIZE_MAX);
+#else
+ grp_buf_len = -1;
+#endif
+ if (grp_buf_len == -1) {
+ grp_buf_len = DEFAULT_GRP_BUF_SZ; /* per the man page */
+ }
+
+ grp_buf = alloca (grp_buf_len);
+ if (!grp_buf) {
+ goto err;
+ }
+
+ getgrgid_r (gid, &grp, grp_buf, grp_buf_len,
+ &grp_result);
+ if (!grp_result)
+ goto err;
+
+ ret = gf_strdup (grp.gr_name);
+ if (ret)
+ return ret;
+ else
+ gf_log (this->name, GF_LOG_ERROR,
+ "gf_strdup failed, failing username "
+ "resolution.");
+err:
+ return ret;
+}
+
+
+/*
+ * This function writes out a latency sample to a given file descriptor
+ * and beautifies the output in the process.
+ */
+void
+_io_stats_write_latency_sample (xlator_t *this, ios_sample_t *sample,
+ FILE *logfp)
+{
+ double epoch_time = 0.00;
+ char *xlator_name = NULL;
+ char *instance_name = NULL;
+ char *hostname = NULL;
+ char *identifier = NULL;
+ char *port = NULL;
+ char *port_pos = NULL;
+ char *group_name = NULL;
+ char *username = NULL;
+ struct ios_conf *conf = NULL;
+
+ conf = this->private;
+
+ epoch_time = (sample->timestamp).tv_sec +
+ ((sample->timestamp).tv_usec / 1000000.0);
+
+ if (!sample->identifier || (strlen (sample->identifier) == 0)) {
+ hostname = "Unknown";
+ port = "Unknown";
+ } else {
+ identifier = strdupa (sample->identifier);
+ port_pos = strrchr (identifier, ':');
+ if (!port_pos || strlen(port_pos) < 2)
+ goto err;
+ port = strdupa (port_pos + 1);
+ if (!port)
+ goto err;
+ *port_pos = '\0';
+ hostname = gf_rev_dns_lookup_cached (identifier,
+ conf->dnscache);
+ if (!hostname)
+ hostname = "Unknown";
+ }
+
+ xlator_name = this->name;
+ if (!xlator_name || strlen (xlator_name) == 0)
+ xlator_name = "Unknown";
+
+ instance_name = this->instance_name;
+ if (!instance_name || strlen (instance_name) == 0)
+ instance_name = "N/A";
+
+ /* Resolve the UID to a string username */
+ username = _resolve_username (this, sample->uid);
+ if (!username) {
+ username = GF_MALLOC (30, gf_common_mt_char);
+ sprintf (username, "%d", (int32_t)sample->uid);
+ }
+
+ /* Resolve the GID to a string group name */
+ group_name = _resolve_group_name (this, sample->gid);
+ if (!group_name) {
+ group_name = GF_MALLOC (30, gf_common_mt_char);
+ sprintf (group_name, "%d", (int32_t)sample->gid);
+ }
+
+ ios_log (this, logfp,
+ "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s",
+ epoch_time, fop_enum_to_pri_string (sample->fop_type),
+ fop_enum_to_string (sample->fop_type),
+ sample->elapsed, xlator_name, instance_name, username,
+ group_name, hostname, port);
+ goto out;
+err:
+ gf_log (this->name, GF_LOG_ERROR,
+ "Error parsing socket identifier");
+out:
+ GF_FREE (group_name);
+ GF_FREE (username);
+}
+
+/*
+ * Takes our current sample buffer in conf->io_sample_buf, and saves
+ * a reference to this, init's a new buffer, and then dumps out the
+ * contents of the saved reference.
+ */
+int
+io_stats_dump_latency_samples_logfp (xlator_t *this, FILE *logfp)
+{
+ uint64_t i = 0;
+ struct ios_conf *conf = NULL;
+ ios_sample_buf_t *sample_buf = NULL;
+ int ret = 1; /* Default to error */
+
+ conf = this->private;
+
+ /* Save pointer to old buffer; the CS equivalent of
+ * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
+ * though ours will end better I hope!
+ */
+ sample_buf = conf->ios_sample_buf;
+ if (!sample_buf) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Sampling buffer is null, bailing!");
+ goto out;
+ }
+
+ /* Empty case, nothing to do, exit. */
+ if (sample_buf->collected == 0) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "No samples, dump not required.");
+ ret = 0;
+ goto out;
+ }
+
+ /* Init a new buffer, so we are free to work on the one we saved a
+ * reference to above.
+ */
+ if (ios_init_sample_buf (conf) != 0) {
+ gf_log (this->name, GF_LOG_WARNING,
+ "Failed to init new sampling buffer, out of memory?");
+ goto out;
+ }
+
+ /* Wrap-around case, dump from pos to sample_buf->size -1
+ * and then from 0 to sample_buf->pos (covered off by
+ * "simple case")
+ */
+ if (sample_buf->collected > sample_buf->pos + 1) {
+ for (i = sample_buf->pos; i < sample_buf->size; i++) {
+ _io_stats_write_latency_sample (this,
+ &(sample_buf->ios_samples[i]), logfp);
+ }
+ }
+
+ /* Simple case: Dump from 0 to sample_buf->pos */
+ for (i = 0; i < sample_buf->pos; i++) {
+ _io_stats_write_latency_sample (this,
+ &(sample_buf->ios_samples[i]), logfp);
+ }
+ ios_destroy_sample_buf (sample_buf);
+
+out:
+ return ret;
+}
+
int
io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
struct timeval *now, int interval, FILE* logfp)
@@ -1284,6 +1604,48 @@ io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
return 0;
}
+void collect_ios_latency_sample (struct ios_conf *conf,
+ glusterfs_fop_t fop_type, double elapsed,
+ call_frame_t *frame)
+{
+ ios_sample_buf_t *ios_sample_buf = NULL;
+ ios_sample_t *ios_sample = NULL;
+ struct timeval *timestamp = NULL;
+ call_stack_t *root = NULL;
+
+
+ ios_sample_buf = conf->ios_sample_buf;
+ LOCK (&conf->ios_sampling_lock);
+ if (conf->ios_sample_interval == 0 ||
+ ios_sample_buf->observed % conf->ios_sample_interval != 0)
+ goto out;
+
+ timestamp = &frame->begin;
+ root = frame->root;
+
+ ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
+ ios_sample->elapsed = elapsed;
+ ios_sample->fop_type = fop_type;
+ ios_sample->uid = root->uid;
+ ios_sample->gid = root->gid;
+ (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
+ (ios_sample->timestamp).tv_usec = timestamp->tv_usec;
+ memcpy (&ios_sample->identifier, &root->identifier,
+ sizeof (root->identifier));
+
+ /* We've reached the end of the circular buffer, start from the
+ * beginning. */
+ if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
+ ios_sample_buf->pos = 0;
+ else
+ ios_sample_buf->pos++;
+ ios_sample_buf->collected++;
+out:
+ ios_sample_buf->observed++;
+ UNLOCK (&conf->ios_sampling_lock);
+ return;
+}
+
static void
update_ios_latency_stats (struct ios_global_stats *stats, double elapsed,
glusterfs_fop_t op)
@@ -1321,6 +1683,7 @@ update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
update_ios_latency_stats (&conf->cumulative, elapsed, op);
update_ios_latency_stats (&conf->incremental, elapsed, op);
+ collect_ios_latency_sample (conf, op, elapsed, frame);
return 0;
}
@@ -1660,7 +2023,7 @@ io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
if (iosstat) {
BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
- iosstat = NULL;
+ iosstat = NULL;
}
STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
@@ -2512,14 +2875,18 @@ _ios_destroy_dump_thread (struct ios_conf *conf) {
void *
_ios_dump_thread (xlator_t *this) {
struct ios_conf *conf = NULL;
- FILE *logfp = NULL;
+ FILE *stats_logfp = NULL;
+ FILE *samples_logfp = NULL;
struct ios_dump_args args = {0};
int i;
- int bytes_written = 0;
- char filename[PATH_MAX];
+ int stats_bytes_written = 0;
+ int samples_bytes_written = 0;
+ char stats_filename[PATH_MAX];
+ char samples_filename[PATH_MAX];
char *xlator_name;
char *instance_name;
- gf_boolean_t log_fopen_failure = _gf_true;
+ gf_boolean_t log_stats_fopen_failure = _gf_true;
+ gf_boolean_t log_samples_fopen_failure = _gf_true;
int old_cancel_type;
conf = this->private;
@@ -2546,17 +2913,34 @@ _ios_dump_thread (xlator_t *this) {
goto out;
}
}
+ if (mkdir (_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+ if (errno != EEXIST) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not create stats-sample directory %s",
+ _IOS_SAMP_DIR);
+ goto out;
+ }
+ }
if (instance_name) {
- bytes_written = snprintf (filename, PATH_MAX,
+ stats_bytes_written = snprintf (stats_filename, PATH_MAX,
"%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
__progname, xlator_name, instance_name);
+ samples_bytes_written = snprintf (samples_filename, PATH_MAX,
+ "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
+ __progname, xlator_name, instance_name);
} else {
- bytes_written = snprintf (filename, PATH_MAX, "%s/%s_%s.dump",
- _IOS_DUMP_DIR, __progname, xlator_name);
- }
- if (bytes_written >= PATH_MAX) {
+ stats_bytes_written = snprintf (stats_filename, PATH_MAX,
+ "%s/%s_%s.dump", _IOS_DUMP_DIR, __progname,
+ xlator_name);
+ samples_bytes_written = snprintf (samples_filename, PATH_MAX,
+ "%s/%s_%s.samp", _IOS_SAMP_DIR, __progname,
+ xlator_name);
+ }
+ if ((stats_bytes_written >= PATH_MAX) ||
+ (samples_bytes_written >= PATH_MAX)) {
gf_log (this->name, GF_LOG_ERROR,
- "Invalid path for IO Stats dump: %s", filename);
+ "Invalid path for stats dump (%s) and/or latency "
+ "samples (%s)", stats_filename, samples_filename);
goto out;
}
while (1) {
@@ -2572,22 +2956,32 @@ _ios_dump_thread (xlator_t *this) {
* just hold it open and rewind/truncate on each iteration.
* Leaving it alone for now.
*/
- logfp = fopen (filename, "w+");
- if (!logfp) {
- if (log_fopen_failure) {
- gf_log (this->name, GF_LOG_ERROR,
- "could not open stats-dump file %s",
- filename);
- log_fopen_failure = _gf_false;
- }
- continue;
+ stats_logfp = fopen (stats_filename, "w+");
+ if (stats_logfp) {
+ (void) ios_dump_args_init (&args,
+ IOS_DUMP_TYPE_JSON_FILE,
+ stats_logfp);
+ io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
+ fclose (stats_logfp);
+ log_stats_fopen_failure = _gf_true;
+ } else if (log_stats_fopen_failure) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not open stats-dump file %s (%s)",
+ stats_filename, strerror(errno));
+ log_stats_fopen_failure = _gf_false;
+ }
+ samples_logfp = fopen (samples_filename, "w+");
+ if (samples_logfp) {
+ io_stats_dump_latency_samples_logfp (this,
+ samples_logfp);
+ fclose (samples_logfp);
+ log_samples_fopen_failure = _gf_true;
+ } else if (log_samples_fopen_failure) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "could not open samples-dump file %s (%s)",
+ samples_filename, strerror(errno));
+ log_samples_fopen_failure = _gf_false;
}
- (void) ios_dump_args_init (
- &args, IOS_DUMP_TYPE_JSON_FILE,
- logfp);
- io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
- fclose (logfp);
- log_fopen_failure = _gf_true;
}
out:
gf_log (this->name, GF_LOG_INFO, "IO stats dump thread terminated");
@@ -3116,6 +3510,10 @@ reconfigure (xlator_t *this, dict_t *options)
(void *) &_ios_dump_thread, this);
}
+ GF_OPTION_RECONF ("ios-sample-interval", conf->ios_sample_interval,
+ options, int32, out);
+ GF_OPTION_RECONF ("ios-sample-buf-size", conf->ios_sample_buf_size,
+ options, int32, out);
GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
if (sys_log_str) {
sys_log_level = glusterd_check_log_level (sys_log_str);
@@ -3229,6 +3627,7 @@ init (xlator_t *this)
* in case of error paths.
*/
LOCK_INIT (&conf->lock);
+ LOCK_INIT (&conf->ios_sampling_lock);
gettimeofday (&conf->cumulative.started_at, NULL);
gettimeofday (&conf->incremental.started_at, NULL);
@@ -3247,6 +3646,22 @@ init (xlator_t *this)
GF_OPTION_INIT ("ios-dump-interval", conf->ios_dump_interval,
int32, out);
+ GF_OPTION_INIT ("ios-sample-interval", conf->ios_sample_interval,
+ int32, out);
+
+ GF_OPTION_INIT ("ios-sample-buf-size", conf->ios_sample_buf_size,
+ int32, out);
+
+ if (ios_init_sample_buf (conf) != 0) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "Out of memory.");
+ return -1;
+ }
+
+ GF_OPTION_INIT ("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec,
+ int32, out);
+ conf->dnscache = gf_dnscache_init (conf->ios_dnscache_ttl_sec);
+
GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
if (sys_log_str) {
sys_log_level = glusterd_check_log_level (sys_log_str);
@@ -3497,6 +3912,29 @@ struct volume_options options[] = {
.description = "Interval (in seconds) at which to auto-dump "
"statistics. Zero disables automatic dumping."
},
+ { .key = { "ios-sample-interval" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 0,
+ .max = 65535,
+ .default_value = "0",
+ .description = "Interval in which we want to collect FOP latency "
+ "samples. 2 means collect a sample every 2nd FOP."
+ },
+ { .key = { "ios-sample-buf-size" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 1024,
+ .max = 1024*1024,
+ .default_value = "65535",
+ .description = "The maximum size of our FOP sampling ring buffer."
+ },
+ { .key = { "ios-dnscache-ttl-sec" },
+ .type = GF_OPTION_TYPE_INT,
+ .min = 1,
+ .max = 3600 * 72,
+ .default_value = "86400",
+ .description = "The interval after wish a cached DNS entry will be "
+ "re-validated. Default: 24 hrs"
+ },
{ .key = { "latency-measurement" },
.type = GF_OPTION_TYPE_BOOL,
.default_value = "off",