summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPoornima G <pgurusid@redhat.com>2016-01-22 11:44:21 -0500
committerKaleb KEITHLEY <kkeithle@redhat.com>2017-01-23 19:13:15 -0500
commitf2e7b6800b812e8bbc9bdbcea4c400a1784e31dc (patch)
tree2809f95a28204a4769e4dac07db8fe7b79a980f9
parent4008a48080b9910b9948c35892e762e91198d134 (diff)
glusterd: add a cli command to trigger a statedump on a client
With this, we will be able to trigger statedumps on remote Gluster clients, mainly targetted for applications using libgfapi. Design: SIGUSR signal is the most comman way of taking a statedump in Gluster. But it cannot be used for libgfapi based processes, as the process loading the library might have already consumed SIGUSR signal. Hence going by the command way. One has to issue a Gluster command to initiate a statedump on the libgfapi based client. The command takes hostname and PID as an argument. All the glusterds in the cluster, check if they are connected to the specified hostname, and send an RPC request to all the connected clients from that hostname (via the mgmt connection). URL: http://review.gluster.org/16357 Change-Id: Icbe4d2f026b32a2c7d5535e1bfb2cdaaff042e91 BUG: 1169302 Signed-off-by: Poornima G <pgurusid@redhat.com> [ndevos: minor fixes and split patch in smaller pieces] Reviewed-on: https://review.gluster.org/9228 Reviewed-by: Niels de Vos <ndevos@redhat.com> Tested-by: Niels de Vos <ndevos@redhat.com> Smoke: Gluster Build System <jenkins@build.gluster.org> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org> Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com> CentOS-regression: Gluster Build System <jenkins@build.gluster.org> Reviewed-by: Samikshan Bairagya <samikshan@gmail.com>
-rw-r--r--cli/src/cli-cmd-parser.c46
-rw-r--r--cli/src/cli-cmd-volume.c4
-rw-r--r--doc/debugging/statedump.md13
-rw-r--r--glusterfsd/src/glusterfsd-mgmt.c1
-rw-r--r--libglusterfs/src/common-utils.c21
-rw-r--r--libglusterfs/src/common-utils.h2
-rw-r--r--rpc/rpc-lib/src/protocol-common.h1
-rw-r--r--rpc/rpc-lib/src/rpc-transport.h2
-rw-r--r--rpc/xdr/src/rpc-common-xdr.x3
-rwxr-xr-xtests/bugs/cli/bug-1169302.t34
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.c43
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-utils.h4
-rw-r--r--xlators/mgmt/glusterd/src/glusterd-volume-ops.c7
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.c67
-rw-r--r--xlators/mgmt/glusterd/src/glusterd.h4
15 files changed, 239 insertions, 13 deletions
diff --git a/cli/src/cli-cmd-parser.c b/cli/src/cli-cmd-parser.c
index 3cc6ca9..d93af0e 100644
--- a/cli/src/cli-cmd-parser.c
+++ b/cli/src/cli-cmd-parser.c
@@ -3541,19 +3541,44 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
dict_t *dict = NULL;
int option_cnt = 0;
char *option = NULL;
- char option_str[100] = {0,};
-
- for (i = 3; i < wordcount; i++, option_cnt++) {
- if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+ char option_str[_POSIX_HOST_NAME_MAX + 100] = {0,};
+ char *tmp = NULL;
+ char *ip_addr = NULL;
+ char *pid = NULL;
+
+ if ((wordcount >= 5) && ((strcmp (words[3], "client")) == 0)) {
+ tmp = gf_strdup(words[4]);
+ if (!tmp) {
+ ret = -1;
+ goto out;
+ }
+ ip_addr = strtok(tmp, ":");
+ pid = strtok(NULL, ":");
+ if (valid_internet_address (ip_addr, _gf_true)
+ && pid && gf_valid_pid (pid, strlen(pid))) {
+ strncat (option_str, words[3], strlen (words[3]));
+ strncat (option_str, " ", 1);
+ strncat (option_str, ip_addr, strlen (ip_addr));
+ strncat (option_str, " ", 1);
+ strncat (option_str, pid, strlen (pid));
+ option_cnt = 3;
+ } else {
+ ret = -1;
+ goto out;
+ }
+ } else {
+ for (i = 3; i < wordcount; i++, option_cnt++) {
+ if (!cli_cmd_validate_dumpoption (words[i], &option)) {
+ ret = -1;
+ goto out;
+ }
+ strncat (option_str, option, strlen (option));
+ strncat (option_str, " ", 1);
+ }
+ if ((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
ret = -1;
goto out;
}
- strncat (option_str, option, strlen (option));
- strncat (option_str, " ", 1);
- }
- if((strstr (option_str, "nfs")) && strstr (option_str, "quotad")) {
- ret = -1;
- goto out;
}
dict = dict_new ();
@@ -3570,6 +3595,7 @@ cli_cmd_volume_statedump_options_parse (const char **words, int wordcount,
*options = dict;
out:
+ GF_FREE (tmp);
if (ret && dict)
dict_unref (dict);
if (ret)
diff --git a/cli/src/cli-cmd-volume.c b/cli/src/cli-cmd-volume.c
index dbf1e3e..5f07a05 100644
--- a/cli/src/cli-cmd-volume.c
+++ b/cli/src/cli-cmd-volume.c
@@ -3285,8 +3285,8 @@ struct cli_cmd volume_cmds[] = {
cli_cmd_volume_heal_cbk,
"self-heal commands on volume specified by <VOLNAME>"},
- {"volume statedump <VOLNAME> [nfs|quotad] [all|mem|iobuf|callpool|priv|fd|"
- "inode|history]...",
+ {"volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|"
+ "priv|fd|inode|history]... | [client <hostname:process-id>]]",
cli_cmd_volume_statedump_cbk,
"perform statedump on bricks"},
diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md
index 18437f1..9939576 100644
--- a/doc/debugging/statedump.md
+++ b/doc/debugging/statedump.md
@@ -19,6 +19,19 @@ For quotad: `gluster volume statedump <volname> quotad`
For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`.
+For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd
+processes could have used the `SIGUSR1` signal already for other purposes.
+To generate statedump for the processes, using libgfapi, below command can be
+executed from one of the nodes in the gluster cluster to which the libgfapi
+application is connected to.
+
+ gluster volume statedump <volname> client <hostname>:<process id>
+
+The statedumps can be found in the `statedump-directory`, the name of the
+statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be
+multiple such files created depending on the number of times the volume is
+accessed by the process (related to the number of `glfs_init()` calls).
+
##How to read statedump
We shall see snippets of each type of statedump.
diff --git a/glusterfsd/src/glusterfsd-mgmt.c b/glusterfsd/src/glusterfsd-mgmt.c
index 92c3343..07c375d 100644
--- a/glusterfsd/src/glusterfsd-mgmt.c
+++ b/glusterfsd/src/glusterfsd-mgmt.c
@@ -1457,6 +1457,7 @@ rpcclnt_cb_actor_t mgmt_cbk_actors[GF_CBK_MAXVALUE] = {
[GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, mgmt_cbk_spec },
[GF_CBK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_CBK_EVENT_NOTIFY,
mgmt_cbk_event},
+ [GF_CBK_STATEDUMP] = {"STATEDUMP", GF_CBK_STATEDUMP, mgmt_cbk_event},
};
diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c
index 18c2a39..0486409 100644
--- a/libglusterfs/src/common-utils.c
+++ b/libglusterfs/src/common-utils.c
@@ -3669,6 +3669,27 @@ out:
return running;
}
+/* Check if the pid is > 0 */
+gf_boolean_t
+gf_valid_pid (const char *pid, int length)
+{
+ gf_boolean_t ret = _gf_true;
+ pid_t value = 0;
+ char *end_ptr = NULL;
+
+ if (length <= 0) {
+ ret = _gf_false;
+ goto out;
+ }
+
+ value = strtol (pid, &end_ptr, 10);
+ if (value <= 0) {
+ ret = _gf_false;
+ }
+out:
+ return ret;
+}
+
static int
dht_is_linkfile_key (dict_t *this, char *key, data_t *value, void *data)
{
diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h
index b77b7ad..14cd696 100644
--- a/libglusterfs/src/common-utils.h
+++ b/libglusterfs/src/common-utils.h
@@ -796,6 +796,8 @@ int gf_thread_create_detached (pthread_t *thread,
gf_boolean_t
gf_is_service_running (char *pidfile, int *pid);
+gf_boolean_t
+gf_valid_pid (const char *pid, int length);
int
gf_skip_header_section (int fd, int header_len);
diff --git a/rpc/rpc-lib/src/protocol-common.h b/rpc/rpc-lib/src/protocol-common.h
index 89a7bb0..69a39b2 100644
--- a/rpc/rpc-lib/src/protocol-common.h
+++ b/rpc/rpc-lib/src/protocol-common.h
@@ -146,6 +146,7 @@ enum gf_cbk_procnum {
GF_CBK_CHILD_UP,
GF_CBK_CHILD_DOWN,
GF_CBK_RECALL_LEASE,
+ GF_CBK_STATEDUMP,
GF_CBK_MAXVALUE,
};
diff --git a/rpc/rpc-lib/src/rpc-transport.h b/rpc/rpc-lib/src/rpc-transport.h
index 717c40a..e3b630e 100644
--- a/rpc/rpc-lib/src/rpc-transport.h
+++ b/rpc/rpc-lib/src/rpc-transport.h
@@ -70,7 +70,7 @@ struct peer_info {
uint32_t max_op_version;
uint32_t min_op_version;
//Volume mounted by client
- char volname[1024];
+ char volname[NAME_MAX];
};
typedef struct peer_info peer_info_t;
diff --git a/rpc/xdr/src/rpc-common-xdr.x b/rpc/xdr/src/rpc-common-xdr.x
index 464a747..7ccfbb1 100644
--- a/rpc/xdr/src/rpc-common-xdr.x
+++ b/rpc/xdr/src/rpc-common-xdr.x
@@ -39,6 +39,9 @@ struct gf_dump_req {
u_quad_t gfs_id;
};
+struct gf_statedump {
+ unsigned int pid;
+};
struct gf_prog_detail {
string progname<>;
diff --git a/tests/bugs/cli/bug-1169302.t b/tests/bugs/cli/bug-1169302.t
new file mode 100755
index 0000000..92252aa
--- /dev/null
+++ b/tests/bugs/cli/bug-1169302.t
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+. $(dirname $0)/../../include.rc
+. $(dirname $0)/../../volume.rc
+. $(dirname $0)/../../cluster.rc
+
+function check_peers {
+ $CLI_1 peer status | grep 'Peer in Cluster (Connected)' | wc -l
+}
+cleanup;
+
+#setup cluster and test volume
+TEST launch_cluster 3; # start 3-node virtual cluster
+TEST $CLI_1 peer probe $H2; # peer probe server 2 from server 1 cli
+TEST $CLI_1 peer probe $H3; # peer probe server 3 from server 1 cli
+
+EXPECT_WITHIN $PROBE_TIMEOUT 2 check_peers;
+
+TEST $CLI_1 volume create $V0 $H1:$B1/$V0 $H2:$B2/$V0 $H3:$B3/$V0
+TEST $CLI_1 volume start $V0
+
+# there is no gfapi application to take statedumps yet, it will get added in
+# the next patch, this only tests the CLI for correctness
+
+cleanup_statedump
+
+TEST ! $CLI_1 volume statedump $V0 client $H2:0
+TEST ! $CLI_2 volume statedump $V0 client $H2:-1
+TEST $CLI_3 volume statedump $V0 client $H2:765
+TEST ! $CLI_1 volume statedump $V0 client $H2:
+TEST ! $CLI_2 volume statedump $V0 client
+
+cleanup_statedump
+cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 2148275..a77cc67 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7044,6 +7044,49 @@ out:
}
int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+ char **op_errstr)
+{
+ int ret = 0;
+ char *dup_options = NULL;
+ char *option = NULL;
+ char *tmpptr = NULL;
+ char msg[256] = {0,};
+ char *target_ip = NULL;
+ char *pid = NULL;
+
+ dup_options = gf_strdup (options);
+ option = strtok_r (dup_options, " ", &tmpptr);
+ if (strcmp (option, "client")) {
+ snprintf (msg, sizeof (msg), "for gluster client statedump, options "
+ "should be after the key 'client'");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+ target_ip = strtok_r (NULL, " ", &tmpptr);
+ if (target_ip == NULL) {
+ snprintf (msg, sizeof (msg), "ip address not specified");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+
+ pid = strtok_r (NULL, " ", &tmpptr);
+ if (pid == NULL) {
+ snprintf (msg, sizeof (msg), "pid not specified");
+ *op_errstr = gf_strdup (msg);
+ ret = -1;
+ goto out;
+ }
+
+ ret = glusterd_client_statedump_submit_req (volname, target_ip, pid);
+out:
+ GF_FREE (dup_options);
+ return ret;
+}
+
+int
glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
{
int ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 5f49053..e801c1a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -390,6 +390,10 @@ int
glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
int
+glusterd_client_statedump (char *volname, char *options, int option_cnt,
+ char **op_errstr);
+
+int
glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
gf_boolean_t
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index 0c3ac58..ecc4f96 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -2867,6 +2867,13 @@ glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
op_errstr);
if (ret)
goto out;
+
+ } else if (strstr (options, "client")) {
+ ret = glusterd_client_statedump (volname, options, option_cnt,
+ op_errstr);
+ if (ret)
+ goto out;
+
} else {
cds_list_for_each_entry (brickinfo, &volinfo->bricks,
brick_list) {
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 18dcf10..d6f8baf 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -47,6 +47,7 @@
#include "glusterd-geo-rep.h"
#include "run.h"
#include "rpc-clnt-ping.h"
+#include "rpc-common-xdr.h"
#include "syncop.h"
@@ -233,6 +234,72 @@ out:
}
int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+ char *pid)
+{
+ gf_statedump statedump_req = {0, };
+ glusterd_conf_t *conf = NULL;
+ int ret = 0;
+ char *end_ptr = NULL;
+ rpc_transport_t *trans = NULL;
+ char *ip_addr = NULL;
+ xlator_t *this = NULL;
+ char tmp[UNIX_PATH_MAX] = {0, };
+
+ this = THIS;
+ GF_ASSERT (this);
+ conf = this->private;
+ GF_ASSERT (conf);
+
+ if (target_ip == NULL || pid == NULL) {
+ ret = -1;
+ goto out;
+ }
+
+ statedump_req.pid = strtol (pid, &end_ptr, 10);
+
+ gf_msg_debug (this->name, 0, "Performing statedump on volume %s "
+ "client with pid:%d host:%s", volname, statedump_req.pid,
+ target_ip);
+
+ pthread_mutex_lock (&conf->xprt_lock);
+ {
+ list_for_each_entry (trans, &conf->xprt_list, list) {
+ /* check if this connection matches "all" or the
+ * volname */
+ if (strncmp (volname, "all", NAME_MAX) &&
+ strncmp (trans->peerinfo.volname, volname,
+ NAME_MAX)) {
+ /* no match, try next trans */
+ continue;
+ }
+
+ strcpy (tmp, trans->peerinfo.identifier);
+ ip_addr = strtok (tmp, ":");
+ if (gf_is_same_address (ip_addr, target_ip)) {
+ /* Every gluster client would have
+ * connected to glusterd(volfile server). This
+ * connection is used to send the statedump
+ * request rpc to the application.
+ */
+ gf_msg_trace (this->name, 0, "Submitting "
+ "statedump rpc request for %s",
+ trans->peerinfo.identifier);
+ rpcsvc_request_submit (conf->rpc, trans,
+ &glusterd_cbk_prog,
+ GF_CBK_STATEDUMP,
+ &statedump_req, this->ctx,
+ (xdrproc_t)xdr_gf_statedump);
+ }
+ }
+ }
+ pthread_mutex_unlock (&conf->xprt_lock);
+out:
+ return ret;
+
+}
+
+int
glusterd_fetchspec_notify (xlator_t *this)
{
int ret = -1;
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 32f2952..d00e4e2 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -1000,6 +1000,10 @@ glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
char *hostname, dict_t *dict);
int
+glusterd_client_statedump_submit_req (char *volname, char *target_ip,
+ char *pid);
+
+int
glusterd_fetchspec_notify (xlator_t *this);
int